Remote runner upload success file using external url

Remote runner can now upload a success file using directly to a given url instead to the server. It avoids the server to have to download the file and upload it again, thus reducing resources usage.
Chocobozzz · Dec 21, 2023 · 29af130 · 29af130
1 parent 00e0d15
commit 29af130
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 4 deletions.
diff --git a/apps/peertube-runner/src/server/process/shared/process-vod.ts b/apps/peertube-runner/src/server/process/shared/process-vod.ts
@@ -119,11 +119,37 @@ export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVO
       videoFile: videoPath
     }
 
+    if (payload.output.videoFileUrl) {
+      logger.info(`Uploading video file to ${payload.output.videoFileUrl.url} for HLS transcoding job ${job.jobToken}`)
+      await server.runnerJobs.uploadTranscodeResult({
+        uploadResultUrl: payload.output.videoFileUrl,
+        file: videoPath,
+        runnerToken,
+        jobToken: job.jobToken,
+        payload: successBody
+      })
+    }
+
+    if (payload.output.playlistFileUrl) {
+      logger.info(`Uploading playlist file to ${payload.output.playlistFileUrl.url} for HLS transcoding job ${job.jobToken}`)
+      await server.runnerJobs.uploadTranscodeResult({
+        uploadResultUrl: payload.output.playlistFileUrl,
+        file: outputPath,
+        runnerToken,
+        jobToken: job.jobToken,
+        payload: successBody
+      })
+    }
+
     await server.runnerJobs.success({
       jobToken: job.jobToken,
       jobUUID: job.uuid,
       runnerToken,
-      payload: successBody
+      payload: successBody,
+      uploadedFiles: {
+        videoFile: !!payload.output.videoFileUrl,
+        resolutionPlaylistFile: !!payload.output.videoFileUrl
+      }
     })
   } finally {
     if (inputPath) await remove(inputPath)

diff --git a/packages/models/src/runners/runner-job-payload.model.ts b/packages/models/src/runners/runner-job-payload.model.ts
@@ -23,12 +23,19 @@ export interface RunnerJobVODWebVideoTranscodingPayload {
   }
 }
 
+export interface UploadUrlTransodingResultPayload {
+  url: string
+  fields: { [key: string]: any } | undefined
+}
+
 export interface RunnerJobVODHLSTranscodingPayload {
   input: {
     videoFileUrl: string
   }
 
   output: {
+    videoFileUrl?: UploadUrlTransodingResultPayload
+    playlistFileUrl?: UploadUrlTransodingResultPayload
     resolution: number
     fps: number
   }

diff --git a/packages/models/src/runners/runner-job-success-body.model.ts b/packages/models/src/runners/runner-job-success-body.model.ts
@@ -3,6 +3,10 @@ export interface RunnerJobSuccessBody {
   jobToken: string
 
   payload: RunnerJobSuccessPayload
+  uploadedFiles?: {
+    videoFile?: boolean
+    resolutionPlaylistFile?: boolean
+  }
 }
 
 // ---------------------------------------------------------------------------

diff --git a/packages/server-commands/src/runners/runner-jobs-command.ts b/packages/server-commands/src/runners/runner-jobs-command.ts
@@ -22,6 +22,7 @@ import {
   RunnerJobType,
   RunnerJobUpdateBody,
   RunnerJobVODPayload,
+  UploadUrlTransodingResultPayload,
   VODHLSTranscodingSuccess,
   VODWebVideoTranscodingSuccess
 } from '@peertube/peertube-models'
@@ -178,19 +179,21 @@ export class RunnerJobsCommand extends AbstractCommand {
   }
 
   success (options: OverrideCommandOptions & RunnerJobSuccessBody & { jobUUID: string }) {
-    const { payload } = options
+    const { payload, uploadedFiles } = options
 
     const path = '/api/v1/runners/jobs/' + options.jobUUID + '/success'
     const attaches: { [id: string]: any } = {}
     let payloadWithoutFiles = payload
 
-    if ((isWebVideoOrAudioMergeTranscodingPayloadSuccess(payload) || isHLSTranscodingPayloadSuccess(payload)) && payload.videoFile) {
+    if (
+      (isWebVideoOrAudioMergeTranscodingPayloadSuccess(payload) || isHLSTranscodingPayloadSuccess(payload)) &&
+      payload.videoFile && !uploadedFiles?.videoFile) {
       attaches[`payload[videoFile]`] = payload.videoFile
 
       payloadWithoutFiles = omit(payloadWithoutFiles as VODWebVideoTranscodingSuccess, [ 'videoFile' ])
     }
 
-    if (isHLSTranscodingPayloadSuccess(payload) && payload.resolutionPlaylistFile) {
+    if (isHLSTranscodingPayloadSuccess(payload) && payload.resolutionPlaylistFile && !uploadedFiles?.resolutionPlaylistFile) {
       attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile
 
       payloadWithoutFiles = omit(payloadWithoutFiles as VODHLSTranscodingSuccess, [ 'resolutionPlaylistFile' ])
@@ -211,6 +214,27 @@ export class RunnerJobsCommand extends AbstractCommand {
     })
   }
 
+  uploadTranscodeResult (options: RunnerJobSuccessBody & { uploadResultUrl: UploadUrlTransodingResultPayload, file: string }) {
+    const { uploadResultUrl, file } = options
+    const parsedUrl = new URL(uploadResultUrl.url)
+    const attaches: { [id: string]: any } = {
+      file
+    }
+
+    return this.postUploadRequest({
+      url: parsedUrl.origin,
+      path: parsedUrl.pathname,
+      fields: {
+        ...uploadResultUrl.fields,
+        ...pick(options, [ 'jobToken', 'runnerToken' ]),
+        file
+      },
+      attaches,
+      implicitToken: false,
+      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  }
+
   getJobFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) {
     const { host, protocol, pathname } = new URL(options.url)