✨ feat: Add audio conversion and transcription functionality

packages/video-uploader/src/audio-queue.ts

@@ -1,11 +1,22 @@
+import { config } from "dotenv";
 import FormData from "form-data";
 import fs from "fs";
+import { MongoClient, ObjectId } from "mongodb";
 import fetch from "node-fetch";
 import { downloadM3U8ToMP3 } from "./utils/ffmpeg";
+import { jsonToVtt, uploadFile } from "./utils/helper";
 import { logger } from "./utils/logger";
 import connection from "./utils/mq";
+config();
 
-const convertAudioToText = async (filepath: string) => {
+const client = new MongoClient(process.env.DB_HOST);
+const db = client.db(process.env.DB_NAME);
+const sessions = db.collection("sessions");
+
+const convertAudioToText = async (
+  sessionId: string,
+  filepath: string,
+): Promise<{ url: string; text: string }> => {
   try {
     const form = new FormData();
     form.append("audio", fs.createReadStream(filepath));
@@ -21,11 +32,21 @@ const convertAudioToText = async (filepath: string) => {
         body: form,
       },
     );
-    return response.json();
+    const data = await response.json();
+    const transcriptions = jsonToVtt(data);
+    const url = await uploadFile(
+      `transcriptions/${sessionId}.vtt`,
+      transcriptions,
+    );
+    return {
+      url: url,
+      text: data.text,
+    };
   } catch (e) {
-    console.error(e);
+    logger.error(e);
   }
 };
+
 async function audioConverter() {
   try {
     const queue = "audio";
@@ -36,25 +57,31 @@ async function audioConverter() {
     channel.consume(
       queue,
       async (msg) => {
-        const payload = Buffer.from(msg.content).toString();
-        const data = JSON.parse(payload);
-        console.log("payload2", data);
-        await downloadM3U8ToMP3(
-          data.session.videoUrl,
-          data.session.slug,
-          "./tmp",
-        );
-        const transcripts = await convertAudioToText(
-          `./tmp/${data.session.slug}.mp3`,
-        );
-        console.log("transcripts", transcripts);
-        fs.unlinkSync(`./tmp/${data.session.slug}.mp3`);
-        // await sessions.findOneAndUpdate(
-        //   { _id: ObjectId.createFromHexString(data.sessionId) },
-        //   {
-        //     audioTranscripts: transcripts,
-        //   }
-        // );
+        try {
+          const payload = Buffer.from(msg.content).toString();
+          const data = JSON.parse(payload);
+          await downloadM3U8ToMP3(
+            data.session.videoUrl,
+            data.session.slug,
+            "./tmp",
+          );
+          const transcript = await convertAudioToText(
+            data.sessionId,
+            `./tmp/${data.session.slug}.mp3`,
+          );
+          await sessions.findOneAndUpdate(
+            { _id: ObjectId.createFromHexString(data.sessionId) },
+            {
+              $set: {
+                videoTranscription: transcript.url,
+                aiDescription: transcript.text,
+              },
+            },
+          );
+          fs.unlinkSync(`./tmp/${data.session.slug}.mp3`);
+        } catch (e) {
+          logger.error("error", e);
+        }
       },
       {
         noAck: true,

packages/video-uploader/src/utils/ffmpeg.ts

@@ -48,7 +48,7 @@ export async function downloadM3U8ToMP3(
   m3u8Url: string,
   fileName: string,
   outputFilePath: string,
-) :Promise<void> {
+): Promise<void> {
   await ensureDirectoryExists(outputFilePath);
   const outputFile = path.join(outputFilePath, `${fileName}.mp3`);
 

packages/video-uploader/src/utils/helper.ts

@@ -0,0 +1,65 @@
+import {
+  PutObjectCommand,
+  PutObjectCommandInput,
+  S3,
+} from "@aws-sdk/client-s3";
+import { config } from "dotenv";
+import { logger } from "./logger";
+config();
+
+interface ChunkTypes {
+  text: string;
+  timestamp: Array<number>;
+}
+
+export interface ChunkDataTypes {
+  chunks: Array<ChunkTypes>;
+  text: string;
+}
+
+const convertToVttTimestamp = (seconds: number): string => {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor((seconds % 3600) / 60);
+  const secs = (seconds % 60).toFixed(3);
+  return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(6, "0")}`;
+};
+
+export const jsonToVtt = ({ chunks }: ChunkDataTypes): string => {
+  const header = "WEBVTT";
+  const body = chunks
+    .map(
+      ({ timestamp: [start, end], text }) =>
+        `${convertToVttTimestamp(start)} --> ${convertToVttTimestamp(end)}
+  ${text.trim()}`,
+    )
+    .join("\n\n");
+  return `${header}\n\n${body}`;
+};
+
+export const uploadFile = async (
+  filename: string,
+  file: string,
+): Promise<string> => {
+  try {
+    const s3 = new S3({
+      endpoint: "https://ams3.digitaloceanspaces.com",
+      region: "us-east-1",
+      credentials: {
+        accessKeyId: process.env.SPACES_KEY ?? "",
+        secretAccessKey: process.env.SPACES_SECRET ?? "",
+      },
+    });
+    const params: PutObjectCommandInput = {
+      Bucket: process.env.BUCKET_NAME,
+      Key: filename,
+      Body: file,
+      ContentType: "text/vtt",
+      ACL: "public-read",
+    };
+    const command = new PutObjectCommand(params);
+    await s3.send(command);
+    return `${process.env.BUCKET_URL}/${filename}`;
+  } catch (e) {
+    logger.error(e);
+  }
+};