Init flow plugins

FlowPluginsTs/FlowHelpers/1.0.0/cliParsers.ts

@@ -0,0 +1,204 @@
+const handbrakeParser = ({
+  str,
+}:
+  {
+    str: string
+  }): number => {
+  if (typeof str !== 'string') {
+    return 0;
+  }
+
+  let percentage = 0;
+  const numbers = '0123456789';
+  const n = str.indexOf('%');
+
+  if (
+    str.length >= 6
+    && str.indexOf('%') >= 6
+    && numbers.includes(str.charAt(n - 5))
+  ) {
+    let output: string = str.substring(n - 6, n + 1);
+    const outputArr: string[] = output.split('');
+    outputArr.splice(outputArr.length - 1, 1);
+    output = outputArr.join('');
+
+    const outputNum = Number(output);
+    if (outputNum > 0) {
+      percentage = outputNum;
+    }
+  }
+
+  return percentage;
+};
+
+// frame=  889 fps=106 q=26.0 Lsize=   25526kB time=00:00:35.69 bitrate=5858.3kbits/s speed=4.25x
+const getFFmpegVar = ({
+  str,
+  variable,
+}: {
+  str: string, variable: string
+}): string => {
+  if (typeof str !== 'string') {
+    return '';
+  }
+
+  const idx = str.indexOf(variable);
+
+  let out = '';
+  let initSpacesEnded = false;
+
+  if (idx >= 0) {
+    const startIdx = idx + variable.length + 1;
+    for (let i = startIdx; i < str.length; i += 1) {
+      if (initSpacesEnded === true && str[i] === ' ') {
+        break;
+      } else if (initSpacesEnded === false && str[i] !== ' ') {
+        initSpacesEnded = true;
+      }
+
+      if (initSpacesEnded === true && str[i] !== ' ') {
+        out += str[i];
+      }
+    }
+  }
+
+  return out;
+};
+
+const getFFmpegPercentage = ({
+  f,
+  fc,
+  vf,
+  d,
+}: {
+
+  f: string, fc: number, vf: number, d: number
+}): number => {
+  let frameCount01: number = fc;
+  let VideoFrameRate: number = vf;
+  let Duration: number = d;
+
+  let perc = 0;
+
+  const frame: number = parseInt(f, 10);
+  frameCount01 = Math.ceil(frameCount01);
+  VideoFrameRate = Math.ceil(VideoFrameRate);
+  Duration = Math.ceil(Duration);
+
+  if (frameCount01 > 0) {
+    perc = ((frame / frameCount01) * 100);
+  } else if (VideoFrameRate > 0 && Duration > 0) {
+    perc = ((frame / (VideoFrameRate * Duration)) * 100);
+  } else {
+    perc = (frame);
+  }
+
+  const percString = perc.toFixed(2);
+
+  // eslint-disable-next-line no-restricted-globals
+  if (isNaN(perc)) {
+    return 0.00;
+  }
+
+  return parseFloat(percString);
+};
+
+const ffmpegParser = ({
+  str,
+  frameCount,
+
+  videoFrameRate,
+  ffprobeDuration,
+  metaDuration,
+}: {
+  str: string,
+  frameCount: number,
+
+  videoFrameRate: number | undefined,
+  ffprobeDuration: string | undefined,
+  metaDuration: number | undefined,
+}): number => {
+  if (typeof str !== 'string') {
+    return 0;
+  }
+
+  let percentage = 0;
+  if (str.length >= 6) {
+    const n = str.indexOf('fps');
+
+    if (n >= 6) {
+      // get frame
+      const frame = getFFmpegVar({
+        str,
+        variable: 'frame',
+      });
+
+      const frameRate = videoFrameRate || 0;
+      let duration = 0;
+
+      if (
+        ffprobeDuration
+        && parseFloat(ffprobeDuration) > 0
+      ) {
+        duration = parseFloat(ffprobeDuration);
+      } else if (metaDuration) {
+        duration = metaDuration;
+      }
+
+      const per = getFFmpegPercentage(
+        {
+          f: frame,
+          fc: frameCount,
+          vf: frameRate,
+          d: duration,
+        },
+      );
+
+      const outputNum = Number(per);
+      if (outputNum > 0) {
+        percentage = outputNum;
+      }
+    }
+  }
+
+  return percentage;
+};
+
+const editreadyParser = ({ str }:{str: string}): number => {
+  if (typeof str !== 'string') {
+    return 0;
+  }
+  let percentage = 0;
+
+  // const ex = 'STATUS: {"progress": "0.0000000"}';
+
+  if (str.includes('STATUS:')) {
+    const parts = str.split('STATUS:');
+
+    if (parts[1]) {
+      try {
+        const json = JSON.parse(parts[1]);
+        const progress = parseFloat(json.progress);
+        const percStr = (progress * 100).toFixed(2);
+        percentage = parseFloat(percStr);
+      } catch (err) {
+        // err
+      }
+    }
+  }
+
+  // eslint-disable-next-line no-restricted-globals
+  if (isNaN(percentage)) {
+    return 0.00;
+  }
+
+  return percentage;
+};
+
+export {
+  handbrakeParser,
+  ffmpegParser,
+  getFFmpegPercentage,
+  getFFmpegVar,
+  editreadyParser,
+};

FlowPluginsTs/FlowHelpers/1.0.0/interfaces/interfaces.ts

@@ -0,0 +1,96 @@
+import { IFileObject, Istreams } from './synced/IFileObject';
+import Ijob from './synced/jobInterface';
+
+export interface IpluginInputUi {
+    type: 'dropdown' | 'text',
+    options: string[],
+}
+
+export interface IpluginInputs {
+    name: string,
+    type: 'string' | 'boolean' | 'number',
+    defaultValue: string,
+    inputUI: IpluginInputUi,
+    tooltip: string,
+}
+
+export interface IpluginDetails {
+    name: string,
+    description: string,
+    style: {
+        borderColor: string,
+        opacity?: number,
+    },
+    tags: string,
+    isStartPlugin: boolean,
+    sidebarPosition: number,
+    icon: string,
+    inputs: IpluginInputs[],
+
+    outputs: {
+        number: number,
+        tooltip: string,
+    }[],
+}
+
+export interface Ilog {
+    (text: string): void
+}
+
+export interface IupdateWorker {
+    (obj: Record<string, unknown>): void,
+}
+
+export interface IffmpegCommandStream extends Istreams {
+    removed: boolean,
+    targetCodec: string,
+    args: string[],
+}
+
+export interface IffmpegCommand {
+    inputFiles: string[],
+    streams: IffmpegCommandStream[]
+    container: string,
+}
+
+export interface Ivariables {
+    ffmpegCommand?: IffmpegCommand
+}
+
+export interface IpluginOutputArgs {
+    outputNumber: number,
+    outputFileObj: {
+        _id: string,
+    },
+    variables: Ivariables
+
+}
+
+export interface IpluginInputArgs {
+    inputFileObj: IFileObject,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    librarySettings: any,
+    inputs: Record<string, unknown>,
+    jobLog: Ilog,
+    workDir: string,
+    platform: string,
+    arch: string,
+    handbrakePath: string,
+    ffmpegPath: string,
+    mkvpropeditPath: string,
+    originalLibraryFile: IFileObject,
+    nodeHardwareType: string,
+    workerType: string,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    config: any,
+    job: Ijob,
+    platform_arch_isdocker: string,
+    variables: Ivariables,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    lastSuccesfulPlugin: any,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    lastSuccessfulRun: any,
+    updateWorker: IupdateWorker,
+    logFullCliOutput: boolean,
+    logOutcome:(outcome:string) => void,
+}

FlowPluginsTs/FlowHelpers/1.0.0/interfaces/synced/IFileObject.ts

@@ -0,0 +1,197 @@
+export interface IstatSync { // tlint-disable-line statSync
+  mtimeMs: number,
+  ctimeMs: number,
+
+  ctime?: '',
+  mtime?: '',
+  atime?: '',
+}
+
+export interface Itags {
+  language?: string,
+  title?: string,
+  [key:string]: string | undefined,
+}
+export interface Istreams {
+  codec_name: string;
+  codec_type?: string,
+  bit_rate?: number,
+  channels?: number,
+  tags?: Itags,
+  avg_frame_rate?: string,
+  nb_frames?: string,
+
+  duration?: number;
+  width?: number,
+  height?: number,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any,@typescript-eslint/explicit-module-boundary-types
+  [index: string]: any
+}
+
+export interface Iformat {
+    'filename'?: string,
+    'nb_streams'?: number,
+    'nb_programs'?: number,
+    'format_name'?: string,
+    'format_long_name'?: string,
+    'start_time'?: string,
+    'duration'?: string,
+    'size'?: string,
+    'bit_rate'?: string,
+    'probe_score'?: number,
+    [key:string]: string | number | undefined
+  }
+
+export interface IffProbeData {
+  streams?: Istreams[]
+  format?: Iformat
+}
+
+export interface Imeta {
+  TrackDuration?: number,
+  MediaDuration?: number,
+  'SourceFile'?: string,
+  'errors'?: [],
+  'Duration'?: number,
+  'ExifToolVersion'?: number,
+  'FileName'?: string,
+  'Directory'?: string,
+  'FileSize'?: string,
+  'FileModifyDate'?: {
+    'year'?: number,
+    'month'?: number,
+    'day'?: number,
+    'hour'?: number,
+    'minute'?: number,
+    'second'?: number,
+    'millisecond'?: number,
+    'tzoffsetMinutes'?: number,
+    'rawValue'?: string,
+  },
+  'FileAccessDate'?: {
+    'year'?: number,
+    'month'?: number,
+    'day'?: number,
+    'hour'?: number,
+    'minute'?: number,
+    'second'?: number,
+    'millisecond'?: number,
+    'tzoffsetMinutes'?: number,
+    'rawValue'?: string,
+  },
+  'FileCreateDate'?: {
+    'year'?: number,
+    'month'?: number,
+    'day'?: number,
+    'hour'?: number,
+    'minute'?: number,
+    'second'?: number,
+    'millisecond'?: number,
+    'tzoffsetMinutes'?: number,
+    'rawValue'?: string,
+  },
+  'FilePermissions'?: string,
+  'FileType'?: string,
+  'FileTypeExtension'?: string,
+  'MIMEType'?: string,
+  'EBMLVersion'?: 1,
+  'EBMLReadVersion'?: 1,
+  'DocType'?: string,
+  'DocTypeVersion'?: 4,
+  'DocTypeReadVersion'?: 2,
+  'TimecodeScale'?: string,
+  'MuxingApp'?: string,
+  'WritingApp'?: string,
+  'VideoFrameRate'?: number,
+  'ImageWidth'?: number,
+  'ImageHeight'?: number,
+  'TrackNumber'?: number,
+  'TrackLanguage'?: string,
+  'CodecID'?: string,
+  'TrackType'?: string,
+  'AudioChannels'?: number,
+  'AudioSampleRate'?: number,
+  'AudioBitsPerSample'?: number,
+  'TagName'?: 'DURATION',
+  'TagString'?: string,
+  'ImageSize'?: string,
+  'Megapixels'?: number,
+}
+
+export interface ImediaInfo {
+  track?: [{
+    '@type': string,
+    'UniqueID': string,
+    'VideoCount': string,
+    'AudioCount': string,
+    'Format': string,
+    'Format_Version': string,
+    'FileSize': string,
+    'Duration': string,
+    'OverallBitRate': string,
+    'FrameRate': string,
+    'FrameCount': string,
+    'IsStreamable': string,
+    'Encoded_Application': string,
+    'Encoded_Library': string,
+    'extra': {
+      'ErrorDetectionType': string,
+    }
+  }],
+}
+
+export interface IFileObjectMin {
+  _id: string,
+  file: string,
+  DB: string,
+  footprintId: string,
+}
+
+type IbaseStatus = '' | 'Hold' | 'Queued'
+export type IHealthCheck = IbaseStatus | 'Success' | 'Error' | 'Cancelled'
+export type ITranscodeDecisionMaker = IbaseStatus | 'Transcode success'
+  | 'Transcode error' | 'Transcode cancelled' | 'Not required'
+
+export interface IFileObjectStripped extends IFileObjectMin {
+  container: string,
+  scannerReads: {
+    ffProbeRead: string,
+  }
+  createdAt: number,
+  lastPluginDetails: string,
+  bit_rate: number,
+  statSync: IstatSync, // tlint-disable-line statSync
+  file_size: number,
+  ffProbeData: IffProbeData,
+  hasClosedCaptions: boolean,
+  bumped: boolean,
+  HealthCheck: IHealthCheck,
+  TranscodeDecisionMaker: ITranscodeDecisionMaker,
+  holdUntil: number,
+  fileMedium: string,
+  video_codec_name: string,
+  audio_codec_name: string,
+  video_resolution: string,
+
+  lastHealthCheckDate: number,
+  lastTranscodeDate: number,
+  history: string,
+  oldSize: number,
+  newSize: number,
+  videoStreamIndex: number;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any,@typescript-eslint/explicit-module-boundary-types
+  [index: string]: any,
+}
+
+export interface IFileObject extends IFileObjectStripped {
+  scannerReads: {
+    ffProbeRead: string,
+    exiftoolRead: string,
+    mediaInfoRead: string,
+    closedCaptionRead: string,
+  }
+  meta?: Imeta,
+  mediaInfo?: ImediaInfo,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any,@typescript-eslint/explicit-module-boundary-types
+  [index: string]: any,
+}

FlowPluginsTs/FlowHelpers/1.0.0/interfaces/synced/jobInterface.ts

@@ -0,0 +1,10 @@
+interface Ijob {
+    version: string,
+    footprintId: string,
+    jobId: string,
+    start: number,
+    type: string,
+    fileId: string
+}
+
+export default Ijob;

FlowPluginsTs/FlowHelpers/1.0.0/utils.ts

@@ -0,0 +1,296 @@
+import { editreadyParser, ffmpegParser, handbrakeParser } from './cliParsers';
+import { Ilog, IupdateWorker } from './interfaces/interfaces';
+import { IFileObject, Istreams } from './interfaces/synced/IFileObject';
+
+const fs = require('fs');
+
+const fancyTimeFormat = (time: number) => {
+  // Hours, minutes and seconds
+
+  // eslint-disable-next-line no-bitwise
+  const hrs = ~~(time / 3600);
+  // eslint-disable-next-line no-bitwise
+  const mins = ~~((time % 3600) / 60);
+  // eslint-disable-next-line no-bitwise
+  const secs = ~~time % 60;
+
+  // Output like "1:01" or "4:03:59" or "123:03:59"
+  let ret = '';
+
+  //  if (hrs > 0) {
+  ret += `${hrs}:${mins < 10 ? '0' : ''}`;
+  // }
+
+  ret += `${mins}:${secs < 10 ? '0' : ''}`;
+  ret += `${secs}`;
+  return ret;
+};
+
+// frame=  889 fps=106 q=26.0 Lsize=   25526kB time=00:00:35.69 bitrate=5858.3kbits/s speed=4.25x
+export const getFFmpegVar = ({
+  str,
+  variable,
+}: {
+  str: string, variable: string
+}): string => {
+  if (typeof str !== 'string') {
+    return '';
+  }
+
+  const idx = str.indexOf(variable);
+
+  let out = '';
+  let initSpacesEnded = false;
+
+  if (idx >= 0) {
+    const startIdx = idx + variable.length + 1;
+    for (let i = startIdx; i < str.length; i += 1) {
+      if (initSpacesEnded === true && str[i] === ' ') {
+        break;
+      } else if (initSpacesEnded === false && str[i] !== ' ') {
+        initSpacesEnded = true;
+      }
+
+      if (initSpacesEnded === true && str[i] !== ' ') {
+        out += str[i];
+      }
+    }
+  }
+
+  return out;
+};
+
+interface Iconfig {
+  cli: string,
+  spawnArgs: string[],
+  spawnOpts: Record<string, unknown>,
+  jobLog: Ilog,
+  outputFilePath: string,
+  updateWorker: IupdateWorker,
+  logFullCliOutput: boolean,
+  inputFileObj: IFileObject,
+}
+
+class CLI {
+  // @ts-expect-error init
+  config: Iconfig = {};
+
+  progAVG: number[] = [];
+
+  oldOutSize = 0;
+
+  oldEstSize = 0;
+
+  oldProgress = 0;
+
+  lastProgCheck = 0;
+
+  constructor(config: Iconfig) {
+    this.config = config;
+  }
+
+  updateETA = (perc: number): void => {
+    if (perc > 0) {
+      if (this.lastProgCheck === 0) {
+        this.lastProgCheck = new Date().getTime();
+        this.oldProgress = perc;
+      } else if (perc !== this.oldProgress) {
+        const n = new Date().getTime();
+        const secsSinceLastCheck = (n - this.lastProgCheck) / 1000;
+
+        if (secsSinceLastCheck > 1) {
+          // eta total
+          let eta = Math.round(
+            (100 / (perc - this.oldProgress)) * secsSinceLastCheck,
+          );
+
+          // eta remaining
+          eta *= ((100 - perc) / 100);
+          this.progAVG.push(eta);
+
+          // let values = [2, 56, 3, 41, 0, 4, 100, 23];
+          const sum = this.progAVG.reduce(
+            // eslint-disable-next-line
+            (previous, current) => (current += previous),
+          );
+          const avg = sum / this.progAVG.length;
+
+          // est size
+          let estSize = 0;
+
+          let outputFileSizeInGbytes;
+
+          try {
+            if (fs.existsSync(this.config.outputFilePath)) {
+              let singleFileSize = fs.statSync(this.config.outputFilePath);
+              singleFileSize = singleFileSize.size;
+              outputFileSizeInGbytes = singleFileSize / (1024 * 1024 * 1024);
+
+              if (outputFileSizeInGbytes !== this.oldOutSize) {
+                this.oldOutSize = outputFileSizeInGbytes;
+                estSize = outputFileSizeInGbytes
+                  + ((100 - perc) / perc) * outputFileSizeInGbytes;
+                this.oldEstSize = estSize;
+              }
+            }
+          } catch (err) {
+            // eslint-disable-next-line no-console
+            console.log(err);
+          }
+
+          this.config.updateWorker({
+            ETA: fancyTimeFormat(avg),
+            outputFileSizeInGbytes: outputFileSizeInGbytes === undefined ? 0 : outputFileSizeInGbytes,
+            estSize: this.oldEstSize === undefined ? 0 : this.oldEstSize,
+          });
+
+          if (this.progAVG.length > 30) {
+            this.progAVG.splice(0, 1);
+          }
+
+          this.lastProgCheck = n;
+          this.oldProgress = perc;
+        }
+      }
+    }
+  };
+
+  parseOutput = (data: string): void => {
+    const str = `${data}`;
+    //
+    if (this.config.logFullCliOutput === true) {
+      this.config.jobLog(str);
+    }
+
+    if (this.config.cli.toLowerCase().includes('handbrake')) {
+      const percentage = handbrakeParser({
+        str,
+      });
+      if (percentage > 0) {
+        this.updateETA(percentage);
+        this.config.updateWorker({
+          percentage,
+        });
+      }
+    } else if (this.config.cli.toLowerCase().includes('ffmpeg')) {
+      const n = str.indexOf('fps');
+      const shouldUpdate = str.length >= 6 && n >= 6;
+
+      const fps = parseInt(getFFmpegVar({
+        str,
+        variable: 'fps',
+      }), 10);
+
+      let frameCount = 0;
+
+      try {
+        // @ts-expect-error type
+        const frameCountTmp = this.config.inputFileObj.ffProbeData?.streams
+          .filter((row: Istreams) => row.codec_type === 'video')[0].nb_frames;
+
+        if (frameCountTmp
+        // @ts-expect-error type
+          && !isNaN(frameCountTmp)) { // eslint-disable-line no-restricted-globals
+          // @ts-expect-error type
+          frameCount = frameCountTmp;
+        }
+      } catch (err) {
+        // err
+      }
+
+      const percentage = ffmpegParser({
+        str,
+        frameCount,
+        videoFrameRate: this.config.inputFileObj?.meta?.VideoFrameRate,
+        ffprobeDuration: this.config.inputFileObj.ffProbeData?.format?.duration,
+        metaDuration: this.config.inputFileObj?.meta?.Duration,
+      });
+
+      if (shouldUpdate === true && fps > 0) {
+        this.config.updateWorker({
+          fps,
+        });
+      }
+
+      if (shouldUpdate === true && percentage > 0) {
+        this.updateETA(percentage);
+        this.config.updateWorker({
+          percentage,
+        });
+      }
+    } else if (this.config.cli.toLowerCase().includes('editready')) {
+      const percentage = editreadyParser({
+        str,
+      });
+      if (percentage > 0) {
+        this.updateETA(percentage);
+        this.config.updateWorker({
+          percentage,
+        });
+      }
+    }
+  };
+
+  runCli = async (): Promise<{
+    cliExitCode: number,
+    errorLogFull: string[],
+  }> => {
+    const childProcess = require('child_process');
+
+    const errorLogFull: string[] = [];
+
+    // eslint-disable-next-line no-console
+    console.log(`Running ${this.config.cli} ${this.config.spawnArgs.join(' ')}`);
+    const cliExitCode: number = await new Promise((resolve) => {
+      try {
+        const opts = this.config.spawnOpts || {};
+        const thread = childProcess.spawn(this.config.cli, this.config.spawnArgs, opts);
+
+        thread.stdout.on('data', (data: string) => {
+          // eslint-disable-next-line no-console
+          console.log(data.toString());
+          errorLogFull.push(data.toString());
+          this.parseOutput(data);
+        });
+
+        thread.stderr.on('data', (data: string) => {
+          // eslint-disable-next-line no-console
+          console.log(data.toString());
+          errorLogFull.push(data.toString());
+          this.parseOutput(data);
+        });
+
+        thread.on('error', () => {
+          // catches execution error (bad file)
+          // eslint-disable-next-line no-console
+          console.log(1, `Error executing binary: ${this.config.cli}`);
+          resolve(1);
+        });
+
+        // thread.stdout.pipe(process.stdout);
+        // thread.stderr.pipe(process.stderr);
+        thread.on('close', (code: number) => {
+          if (code !== 0) {
+            // eslint-disable-next-line no-console
+            console.log(code, 'FFmpeg error');
+          }
+          resolve(code);
+        });
+      } catch (err) {
+        // catches execution error (no file)
+        // eslint-disable-next-line no-console
+        console.log(1, `Error executing binary: ${this.config.cli}`);
+        resolve(1);
+      }
+    });
+
+    return {
+      cliExitCode,
+      errorLogFull,
+    };
+  };
+}
+
+export {
+  CLI,
+};