add chat service

packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/Transcipts/index.tsx

@@ -1,6 +1,6 @@
 "use client"
 import { useClipContext } from '../../ClipContext';
-
+import { Button } from '@/components/ui/button'; 
 const Transcripts = ({
   words,
 }: {
@@ -19,6 +19,7 @@ const Transcripts = ({
 
   return (
     <div className="whitespace-pre-wrap">
+      <Button>Extract Highlights</Button>
       {words?.map((word, index) => (
         <span
           key={`${word.word}-${index}`}

packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/clips/Clip.tsx

@@ -4,12 +4,11 @@ import { Card, CardContent } from '@/components/ui/card';
 import { fetchAsset } from '@/lib/services/sessionService';
 import { IExtendedSession } from '@/lib/types';
 import { formatDate } from '@/lib/utils/time';
-import { Asset } from 'livepeer/models/components';
 import { useRouter } from 'next/navigation';
 import { useEffect, useState } from 'react';
 import { ProcessingStatus } from 'streameth-new-server/src/interfaces/session.interface';
 import Preview from './Preview';
-
+import { Asset } from 'livepeer/models/components/asset';
 export default function Clip({ session }: { session: IExtendedSession }) {
   const { name, coverImage, assetId } = session;
   const [asset, setAsset] = useState<Asset | null>(null);

packages/app/app/studio/[organization]/(root)/library/components/TableCells.tsx

@@ -106,7 +106,7 @@ const TableCells = async ({
       </TableCell>
       <TableCell className={rowBackgroundClass}>
         <div className="flex justify-end items-center space-x-2 max-w-[100px]">
-          {!isDisabled && item.type === 'livestream' && (
+          {!isDisabled && (
             // item.createdAt &&
             // new Date(item.createdAt).getTime() >
             //   Date.now() - 7 * 24 * 60 * 60 * 1000 && (

packages/app/lib/actions/sessions.ts

@@ -14,6 +14,7 @@ import {
   saveSessionImport,
   generateTranscriptions,
   uploadSessionToSocialsRequest,
+  extractHighlights,
 } from '../services/sessionService';
 import {
   ISession,
@@ -280,3 +281,13 @@ export const generateTranscriptionActions = async ({
     return null;
   }
 };
+
+
+export const extractHighlightsAction = async ({
+  sessionId,
+}: {
+  sessionId: string;
+}) => {
+  const res = await extractHighlights({ sessionId });
+  return res;
+};

packages/app/lib/services/sessionService.ts

@@ -7,7 +7,7 @@ import {
 import { apiUrl } from '@/lib/utils/utils';
 import { ISession } from 'streameth-new-server/src/interfaces/session.interface';
 import { revalidatePath } from 'next/cache';
-import { Asset } from 'livepeer/models/components';
+import { Asset } from 'livepeer/models/components/asset';
 import FuzzySearch from 'fuzzy-search';
 import { fetchClient } from './fetch-client';
 
@@ -558,3 +558,18 @@ export const generateTranscriptions = async ({
     throw e;
   }
 };
+
+export const extractHighlights = async ({
+  sessionId,
+}: {
+  sessionId: string;
+}) => {
+  try {
+    const response = await fetchClient(`${apiUrl()}/sessions/${sessionId}/highlights`, {
+      method: 'POST',
+    });
+  } catch (e) {
+    console.log('error in extractHighlights', e);
+    throw e;
+  }
+};

packages/server/package.json

@@ -40,6 +40,7 @@
     "fuse.js": "^7.0.0",
     "google-auth-library": "^9.6.3",
     "googleapis": "^133.0.0",
+    "gpt-3-encoder": "^1.1.4",
     "helmet": "^7.1.0",
     "hpp": "^0.2.3",
     "http-errors": "^2.0.0",

packages/server/src/controllers/session.controller.ts

@@ -199,4 +199,18 @@ export class SessionController extends Controller {
     await this.sessionService.deleteOne(sessionId);
     return SendApiResponse('deleted');
   }
+
+  /**
+ * @summary Extract highlights from session
+   */
+  @SuccessResponse('200')
+  @Post('{sessionId}/highlights')
+  async extractHighlights(
+    @Path() sessionId: string,
+  ): Promise<IStandardResponse<void>> {
+    const highlights = await this.sessionService.extractHighlights(sessionId);
+    return SendApiResponse(highlights.content);
+  }
 }
+
+

packages/server/src/routes/routes.ts

@@ -2734,6 +2734,36 @@ export function RegisterRoutes(app: Router,opts?:{multer?:ReturnType<typeof mult
             }
         });
         // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa
+        app.post('/sessions/:sessionId/highlights',
+            ...(fetchMiddlewares<RequestHandler>(SessionController)),
+            ...(fetchMiddlewares<RequestHandler>(SessionController.prototype.extractHighlights)),
+
+            async function SessionController_extractHighlights(request: ExRequest, response: ExResponse, next: any) {
+            const args: Record<string, TsoaRoute.ParameterSchema> = {
+                    sessionId: {"in":"path","name":"sessionId","required":true,"dataType":"string"},
+            };
+
+            // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa
+
+            let validatedArgs: any[] = [];
+            try {
+                validatedArgs = templateService.getValidatedArgs({ args, request, response });
+
+                const controller = new SessionController();
+
+              await templateService.apiHandler({
+                methodName: 'extractHighlights',
+                controller,
+                response,
+                next,
+                validatedArgs,
+                successStatus: 200,
+              });
+            } catch (err) {
+                return next(err);
+            }
+        });
+        // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa
         app.post('/schedule/import',
             authenticateMiddleware([{"jwt":["org"]}]),
             ...(fetchMiddlewares<RequestHandler>(ScheduleImporterController)),

packages/server/src/services/session.service.ts

@@ -19,6 +19,7 @@ import Organization from '@models/organization.model';
 import Session from '@models/session.model';
 import Stage from '@models/stage.model';
 import State from '@models/state.model';
+import { ChatAPI } from '@utils/ai.chat';
 import { getAsset, getDownloadUrl, getStreamRecordings } from '@utils/livepeer';
 import { refreshAccessToken } from '@utils/oauth';
 import { sessionTranscriptionsQueue, videoUploadQueue } from '@utils/redis';
@@ -393,4 +394,52 @@ export default class SessionService {
     const query = isObjectId ? { _id: id } : { slug: id };
     return query;
   }
+
+  async extractHighlights(sessionId: string) {
+    const session = await this.get(sessionId);
+
+    if (!session.transcripts) {
+      throw new HttpException(400, 'Session has no transcripts');
+    }
+
+    const chunks = session.transcripts.chunks;
+    const chunkSlices = [
+      chunks.slice(0, Math.floor(chunks.length / 3)),
+      chunks.slice(
+        Math.floor(chunks.length / 3),
+        Math.floor(chunks.length / 3) * 2,
+      ),
+      chunks.slice(Math.floor(chunks.length / 3) * 2, chunks.length),
+    ];
+
+    for (let i = 0; i < 1; i++) {
+      const chat = new ChatAPI();
+      const highlights = await chat.chat([
+        {
+          role: 'system',
+          content: `
+          You are an expert video editor specializing in creating highlights optimized for social media platforms like TikTok, X, and Instagram.
+          Task: Extract segments from the transcript that are 30 to 120 seconds long and are the most engaging and impactful moments of the event that are related to ethereum technology.
+          Input: You will receive an array of words with timestamps from the English transcript.
+          Output: Return a JSON array of objects with the following structure:
+          {
+            "start": number,    // Timestamp when highlight begins
+            "end": number,      // Timestamp when highlight ends
+            "full_transcript": string  // Complete transcript of the highlighted segment
+          }
+start-ends should be 60 to 120 seconds long
+          Guidelines:
+          - Select engaging, impactful moments that will resonate on social media
+          - Each highlight should be 60-120 seconds long
+          - Focus on key technical insights, announcements, or memorable quotes
+          - Ensure the selected segments are self-contained and make sense standalone`,
+        },
+        {
+          role: 'user',
+          content: `Here is the transcript: ${chunkSlices[i]}`,
+        },
+      ]);
+      console.log(highlights);
+    }
+  }
 }

packages/server/src/services/stage.service.ts

@@ -226,4 +226,7 @@ export default class StageService {
     });
     return stream;
   }
-}
+
+
+}
+

packages/server/src/swagger/swagger.json

@@ -6357,6 +6357,38 @@
 				}
 			}
 		},
+		"/sessions/{sessionId}/highlights": {
+			"post": {
+				"operationId": "ExtractHighlights",
+				"responses": {
+					"200": {
+						"description": "",
+						"content": {
+							"application/json": {
+								"schema": {
+									"$ref": "#/components/schemas/IStandardResponse_void_"
+								}
+							}
+						}
+					}
+				},
+				"summary": "Extract highlights from session",
+				"tags": [
+					"Session"
+				],
+				"security": [],
+				"parameters": [
+					{
+						"in": "path",
+						"name": "sessionId",
+						"required": true,
+						"schema": {
+							"type": "string"
+						}
+					}
+				]
+			}
+		},
 		"/schedule/import": {
 			"post": {
 				"operationId": "ImportSchdeule",

packages/server/src/utils/ai.chat.ts

@@ -0,0 +1,30 @@
+import OpenAI from 'openai';
+import { config } from '@config';
+import { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
+
+export class ChatAPI {
+  private openai: OpenAI;
+  private maxTokens: number;
+
+  constructor(maxTokens: number = 12800) {
+    this.openai = new OpenAI({
+      apiKey: config.openai.apiKey,
+    });
+    this.maxTokens = maxTokens;
+  }
+
+  async chat(messages: ChatCompletionMessageParam[]) {
+
+
+
+
+    const completion = await this.openai.chat.completions.create({
+      model: 'gpt-4o-mini',
+      messages,
+      response_format: { type: 'json_object' },
+    });
+    return completion.choices[0].message;
+  }
+
+
+}

packages/server/workers/session-transcriptions/index.ts

@@ -1,7 +1,7 @@
 import 'dotenv/config';
 import { sessionTranscriptionsQueue } from '@utils/redis';
 import ffmpeg from 'fluent-ffmpeg';
-import WhisperAPI from '@utils/whisper';
+import WhisperAPI from '@utils/ai.transcribes';
 import { dbConnection } from '@databases/index';
 import { connect } from 'mongoose';
 import { tmpdir } from 'os';

packages/server/workers/stage-transcriptions/index.ts

@@ -2,7 +2,7 @@ import { stageTranscriptionsQueue } from '@utils/redis';
 import ffmpeg from 'fluent-ffmpeg';
 import StageService from '@services/stage.service';
 import { buildPlaybackUrl } from '@utils/livepeer';
-import WhisperAPI from '@utils/whisper';
+import WhisperAPI from '@utils/ai.transcribes';
 import { IStage } from '@interfaces/stage.interface';
 import { dbConnection } from '@databases/index';
 import { connect } from 'mongoose';

yarn.lock

@@ -15009,6 +15009,11 @@ got@^11.8.5:
     p-cancelable "^2.0.0"
     responselike "^2.0.0"
 
+gpt-3-encoder@^1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/gpt-3-encoder/-/gpt-3-encoder-1.1.4.tgz#d6cdaacf5824857e133b6065247c757fc7e4fa72"
+  integrity sha512-fSQRePV+HUAhCn7+7HL7lNIXNm6eaFWFbNLOOGtmSJ0qJycyQvj60OvRlH7mee8xAMjBDNRdMXlMwjAbMTDjkg==
+
 graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9:
   version "4.2.11"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"