From 921b78a4c0a6d3611531b6bd544ba8264dc4e5ef Mon Sep 17 00:00:00 2001 From: Pablo Voorvaart Date: Thu, 9 Jan 2025 12:27:27 +0100 Subject: [PATCH] add chat service --- .../[stageId]/sidebar/Transcipts/index.tsx | 3 +- .../clips/[stageId]/sidebar/clips/Clip.tsx | 3 +- .../(root)/library/components/TableCells.tsx | 2 +- packages/app/lib/actions/sessions.ts | 11 +++++ packages/app/lib/services/sessionService.ts | 17 ++++++- packages/server/package.json | 1 + .../src/controllers/session.controller.ts | 14 ++++++ packages/server/src/routes/routes.ts | 30 ++++++++++++ .../server/src/services/session.service.ts | 49 +++++++++++++++++++ packages/server/src/services/stage.service.ts | 5 +- packages/server/src/swagger/swagger.json | 32 ++++++++++++ packages/server/src/utils/ai.chat.ts | 30 ++++++++++++ .../utils/{whisper.ts => ai.transcribes.ts} | 0 .../workers/session-transcriptions/index.ts | 2 +- .../workers/stage-transcriptions/index.ts | 2 +- yarn.lock | 5 ++ 16 files changed, 198 insertions(+), 8 deletions(-) create mode 100644 packages/server/src/utils/ai.chat.ts rename packages/server/src/utils/{whisper.ts => ai.transcribes.ts} (100%) diff --git a/packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/Transcipts/index.tsx b/packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/Transcipts/index.tsx index 26d029afe..f0d2d8727 100644 --- a/packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/Transcipts/index.tsx +++ b/packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/Transcipts/index.tsx @@ -1,6 +1,6 @@ "use client" import { useClipContext } from '../../ClipContext'; - +import { Button } from '@/components/ui/button'; const Transcripts = ({ words, }: { @@ -19,6 +19,7 @@ const Transcripts = ({ return (
+ {words?.map((word, index) => ( (null); diff --git a/packages/app/app/studio/[organization]/(root)/library/components/TableCells.tsx b/packages/app/app/studio/[organization]/(root)/library/components/TableCells.tsx index 4f11da293..855916dd3 100644 --- a/packages/app/app/studio/[organization]/(root)/library/components/TableCells.tsx +++ b/packages/app/app/studio/[organization]/(root)/library/components/TableCells.tsx @@ -106,7 +106,7 @@ const TableCells = async ({
- {!isDisabled && item.type === 'livestream' && ( + {!isDisabled && ( // item.createdAt && // new Date(item.createdAt).getTime() > // Date.now() - 7 * 24 * 60 * 60 * 1000 && ( diff --git a/packages/app/lib/actions/sessions.ts b/packages/app/lib/actions/sessions.ts index bcefb47c9..5deb44a88 100644 --- a/packages/app/lib/actions/sessions.ts +++ b/packages/app/lib/actions/sessions.ts @@ -14,6 +14,7 @@ import { saveSessionImport, generateTranscriptions, uploadSessionToSocialsRequest, + extractHighlights, } from '../services/sessionService'; import { ISession, @@ -280,3 +281,13 @@ export const generateTranscriptionActions = async ({ return null; } }; + + +export const extractHighlightsAction = async ({ + sessionId, +}: { + sessionId: string; +}) => { + const res = await extractHighlights({ sessionId }); + return res; +}; diff --git a/packages/app/lib/services/sessionService.ts b/packages/app/lib/services/sessionService.ts index 32bd39448..392572dc8 100644 --- a/packages/app/lib/services/sessionService.ts +++ b/packages/app/lib/services/sessionService.ts @@ -7,7 +7,7 @@ import { import { apiUrl } from '@/lib/utils/utils'; import { ISession } from 'streameth-new-server/src/interfaces/session.interface'; import { revalidatePath } from 'next/cache'; -import { Asset } from 'livepeer/models/components'; +import { Asset } from 'livepeer/models/components/asset'; import FuzzySearch from 'fuzzy-search'; import { fetchClient } from './fetch-client'; @@ -558,3 +558,18 @@ export const generateTranscriptions = async ({ throw e; } }; + +export const extractHighlights = async ({ + sessionId, +}: { + sessionId: string; +}) => { + try { + const response = await fetchClient(`${apiUrl()}/sessions/${sessionId}/highlights`, { + method: 'POST', + }); + } catch (e) { + console.log('error in extractHighlights', e); + throw e; + } +}; diff --git a/packages/server/package.json b/packages/server/package.json index 9a6068871..444be7ef9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -40,6 +40,7 @@ "fuse.js": "^7.0.0", "google-auth-library": "^9.6.3", "googleapis": "^133.0.0", + "gpt-3-encoder": "^1.1.4", "helmet": "^7.1.0", "hpp": "^0.2.3", "http-errors": "^2.0.0", diff --git a/packages/server/src/controllers/session.controller.ts b/packages/server/src/controllers/session.controller.ts index 88bea2067..e6124725d 100644 --- a/packages/server/src/controllers/session.controller.ts +++ b/packages/server/src/controllers/session.controller.ts @@ -199,4 +199,18 @@ export class SessionController extends Controller { await this.sessionService.deleteOne(sessionId); return SendApiResponse('deleted'); } + + /** + * @summary Extract highlights from session + */ + @SuccessResponse('200') + @Post('{sessionId}/highlights') + async extractHighlights( + @Path() sessionId: string, + ): Promise> { + const highlights = await this.sessionService.extractHighlights(sessionId); + return SendApiResponse(highlights.content); + } } + + diff --git a/packages/server/src/routes/routes.ts b/packages/server/src/routes/routes.ts index dce8f9753..db0169287 100644 --- a/packages/server/src/routes/routes.ts +++ b/packages/server/src/routes/routes.ts @@ -2734,6 +2734,36 @@ export function RegisterRoutes(app: Router,opts?:{multer?:ReturnType(SessionController)), + ...(fetchMiddlewares(SessionController.prototype.extractHighlights)), + + async function SessionController_extractHighlights(request: ExRequest, response: ExResponse, next: any) { + const args: Record = { + sessionId: {"in":"path","name":"sessionId","required":true,"dataType":"string"}, + }; + + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa + + let validatedArgs: any[] = []; + try { + validatedArgs = templateService.getValidatedArgs({ args, request, response }); + + const controller = new SessionController(); + + await templateService.apiHandler({ + methodName: 'extractHighlights', + controller, + response, + next, + validatedArgs, + successStatus: 200, + }); + } catch (err) { + return next(err); + } + }); + // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa app.post('/schedule/import', authenticateMiddleware([{"jwt":["org"]}]), ...(fetchMiddlewares(ScheduleImporterController)), diff --git a/packages/server/src/services/session.service.ts b/packages/server/src/services/session.service.ts index c2d8c6d72..0d58e744b 100644 --- a/packages/server/src/services/session.service.ts +++ b/packages/server/src/services/session.service.ts @@ -19,6 +19,7 @@ import Organization from '@models/organization.model'; import Session from '@models/session.model'; import Stage from '@models/stage.model'; import State from '@models/state.model'; +import { ChatAPI } from '@utils/ai.chat'; import { getAsset, getDownloadUrl, getStreamRecordings } from '@utils/livepeer'; import { refreshAccessToken } from '@utils/oauth'; import { sessionTranscriptionsQueue, videoUploadQueue } from '@utils/redis'; @@ -393,4 +394,52 @@ export default class SessionService { const query = isObjectId ? { _id: id } : { slug: id }; return query; } + + async extractHighlights(sessionId: string) { + const session = await this.get(sessionId); + + if (!session.transcripts) { + throw new HttpException(400, 'Session has no transcripts'); + } + + const chunks = session.transcripts.chunks; + const chunkSlices = [ + chunks.slice(0, Math.floor(chunks.length / 3)), + chunks.slice( + Math.floor(chunks.length / 3), + Math.floor(chunks.length / 3) * 2, + ), + chunks.slice(Math.floor(chunks.length / 3) * 2, chunks.length), + ]; + + for (let i = 0; i < 1; i++) { + const chat = new ChatAPI(); + const highlights = await chat.chat([ + { + role: 'system', + content: ` + You are an expert video editor specializing in creating highlights optimized for social media platforms like TikTok, X, and Instagram. + Task: Extract segments from the transcript that are 30 to 120 seconds long and are the most engaging and impactful moments of the event that are related to ethereum technology. + Input: You will receive an array of words with timestamps from the English transcript. + Output: Return a JSON array of objects with the following structure: + { + "start": number, // Timestamp when highlight begins + "end": number, // Timestamp when highlight ends + "full_transcript": string // Complete transcript of the highlighted segment + } +start-ends should be 60 to 120 seconds long + Guidelines: + - Select engaging, impactful moments that will resonate on social media + - Each highlight should be 60-120 seconds long + - Focus on key technical insights, announcements, or memorable quotes + - Ensure the selected segments are self-contained and make sense standalone`, + }, + { + role: 'user', + content: `Here is the transcript: ${chunkSlices[i]}`, + }, + ]); + console.log(highlights); + } + } } diff --git a/packages/server/src/services/stage.service.ts b/packages/server/src/services/stage.service.ts index f031cc787..88cd72090 100644 --- a/packages/server/src/services/stage.service.ts +++ b/packages/server/src/services/stage.service.ts @@ -226,4 +226,7 @@ export default class StageService { }); return stream; } -} \ No newline at end of file + + +} + diff --git a/packages/server/src/swagger/swagger.json b/packages/server/src/swagger/swagger.json index 9ac704d65..dedbc3405 100644 --- a/packages/server/src/swagger/swagger.json +++ b/packages/server/src/swagger/swagger.json @@ -6357,6 +6357,38 @@ } } }, + "/sessions/{sessionId}/highlights": { + "post": { + "operationId": "ExtractHighlights", + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IStandardResponse_void_" + } + } + } + } + }, + "summary": "Extract highlights from session", + "tags": [ + "Session" + ], + "security": [], + "parameters": [ + { + "in": "path", + "name": "sessionId", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, "/schedule/import": { "post": { "operationId": "ImportSchdeule", diff --git a/packages/server/src/utils/ai.chat.ts b/packages/server/src/utils/ai.chat.ts new file mode 100644 index 000000000..735a18b58 --- /dev/null +++ b/packages/server/src/utils/ai.chat.ts @@ -0,0 +1,30 @@ +import OpenAI from 'openai'; +import { config } from '@config'; +import { ChatCompletionMessageParam } from 'openai/resources/chat/completions'; + +export class ChatAPI { + private openai: OpenAI; + private maxTokens: number; + + constructor(maxTokens: number = 12800) { + this.openai = new OpenAI({ + apiKey: config.openai.apiKey, + }); + this.maxTokens = maxTokens; + } + + async chat(messages: ChatCompletionMessageParam[]) { + + + + + const completion = await this.openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages, + response_format: { type: 'json_object' }, + }); + return completion.choices[0].message; + } + + +} diff --git a/packages/server/src/utils/whisper.ts b/packages/server/src/utils/ai.transcribes.ts similarity index 100% rename from packages/server/src/utils/whisper.ts rename to packages/server/src/utils/ai.transcribes.ts diff --git a/packages/server/workers/session-transcriptions/index.ts b/packages/server/workers/session-transcriptions/index.ts index 423239539..1e4070757 100644 --- a/packages/server/workers/session-transcriptions/index.ts +++ b/packages/server/workers/session-transcriptions/index.ts @@ -1,7 +1,7 @@ import 'dotenv/config'; import { sessionTranscriptionsQueue } from '@utils/redis'; import ffmpeg from 'fluent-ffmpeg'; -import WhisperAPI from '@utils/whisper'; +import WhisperAPI from '@utils/ai.transcribes'; import { dbConnection } from '@databases/index'; import { connect } from 'mongoose'; import { tmpdir } from 'os'; diff --git a/packages/server/workers/stage-transcriptions/index.ts b/packages/server/workers/stage-transcriptions/index.ts index cc7efd9e8..a6f24db37 100644 --- a/packages/server/workers/stage-transcriptions/index.ts +++ b/packages/server/workers/stage-transcriptions/index.ts @@ -2,7 +2,7 @@ import { stageTranscriptionsQueue } from '@utils/redis'; import ffmpeg from 'fluent-ffmpeg'; import StageService from '@services/stage.service'; import { buildPlaybackUrl } from '@utils/livepeer'; -import WhisperAPI from '@utils/whisper'; +import WhisperAPI from '@utils/ai.transcribes'; import { IStage } from '@interfaces/stage.interface'; import { dbConnection } from '@databases/index'; import { connect } from 'mongoose'; diff --git a/yarn.lock b/yarn.lock index 33f3fcc73..a99e1c36e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15009,6 +15009,11 @@ got@^11.8.5: p-cancelable "^2.0.0" responselike "^2.0.0" +gpt-3-encoder@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/gpt-3-encoder/-/gpt-3-encoder-1.1.4.tgz#d6cdaacf5824857e133b6065247c757fc7e4fa72" + integrity sha512-fSQRePV+HUAhCn7+7HL7lNIXNm6eaFWFbNLOOGtmSJ0qJycyQvj60OvRlH7mee8xAMjBDNRdMXlMwjAbMTDjkg== + graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"