Merge pull request #1139 from streamethorg/ai-transcribes

Ai transcribes

packages/app/app/[organization]/layout.tsx

@@ -33,6 +33,7 @@ const Layout = async ({
 
   const userData = await fetchUserAction();
 
+  console.log(organization);
   if (!organization) {
     return NotFound();
   }

packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/ClipContext.tsx

@@ -103,10 +103,8 @@ export const ClipProvider = ({
   organizationId: string;
   clipUrl: string;
 }) => {
-  const { handleTermChange, searchParams } = useSearchParams();
+  const { searchParams } = useSearchParams();
 
-  const start = searchParams?.get('start');
-  const end = searchParams?.get('end');
   const [playbackStatus, setPlaybackStatus] = useState<PlaybackStatus | null>(
     null
   );

packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/page.tsx

@@ -1,9 +1,7 @@
 import { fetchOrganization } from '@/lib/services/organizationService';
 import {
   fetchAllSessions,
-  fetchAsset,
-  fetchSession,
-  sessionImport,
+  fetchSession
 } from '@/lib/services/sessionService';
 import { fetchStage, fetchStageRecordings } from '@/lib/services/stageService';
 import { ClipsPageParams } from '@/lib/types';
@@ -31,12 +29,11 @@ const fetchVideoDetails = async (
 
       const stageRecordings = await fetchStageRecordings({ streamId });
       if (!stageRecordings?.recordings[0]) return null;
-
       return {
-        videoSrc: stageRecordings.recordings[0].recordingUrl,
+        videoSrc: `https://livepeercdn.studio/hls/${liveStage.streamSettings?.playbackId}/index.m3u8`,
         type: 'livepeer',
         name: liveStage.name,
-        words: liveStage.transcripts?.text,
+        words: liveStage.transcripts?.chunks,
         liveRecording: stageRecordings.recordings[0],
       };
     }
@@ -47,13 +44,13 @@ const fetchVideoDetails = async (
 
       const stage = await fetchStage({ stage: session.stageId as string });
       if (!stage?.streamSettings?.playbackId) return null;
-
+      console.log('session', session.transcripts?.chunks);
       const videoSrc = await getVideoUrlAction(session);
       return {
         videoSrc,
         type: 'livepeer',
         name: session.name,
-        words: session.transcripts?.subtitleUrl,
+        words: session.transcripts?.chunks,
       };
     }
 
@@ -65,6 +62,7 @@ const fetchVideoDetails = async (
         videoSrc: stage.source.m3u8Url,
         type: stage.source.type,
         name: stage.name,
+        words: stage.transcripts?.chunks,
       };
     }
 
@@ -106,13 +104,6 @@ const ClipsConfig = async ({ params, searchParams }: ClipsPageParams) => {
       clipUrl={videoDetails.videoSrc}
     >
       <div className="flex flex-row w-full h-full border-t border-gray-200">
-        {/* <div className="flex flex-col w-[400px] h-full overflow-y-auto">
-          {words?.split('\n').map((word) => (
-            <div className="flex flex-col text-sm text-gray-500" key={word}>
-              {word}
-            </div>
-          ))}
-        </div> */}
         <div className="flex h-full w-[calc(100%-400px)] flex-col">
           <TopBar title={videoDetails.name} organization={organization} />
           <ReactHlsPlayer
@@ -130,6 +121,7 @@ const ClipsConfig = async ({ params, searchParams }: ClipsPageParams) => {
             stageSessions={stageSessions.sessions}
             organizationId={organizationId}
             animations={animations.sessions}
+            words={videoDetails.words}
           />
         </div>
       </div>

packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/Transcipts/index.tsx

@@ -0,0 +1,42 @@
+"use client"
+import { useClipContext } from '../../ClipContext';
+import { Button } from '@/components/ui/button'; 
+const Transcripts = ({
+  words,
+}: {
+  words: { word: string; start: number }[];
+}) => {
+  const { currentTime, videoRef } = useClipContext();
+
+  // Helper function to determine if a word should be highlighted
+  const isWordActive = (
+    word: { word: string; start: number },
+    currentTime: number
+  ) => {
+    // You might want to adjust this logic based on your requirements
+    return word.start <= currentTime && word.start + 1 > currentTime;
+  };
+
+  return (
+    <div className="whitespace-pre-wrap">
+      <Button>Extract Highlights</Button>
+      {words?.map((word, index) => (
+        <span
+          key={`${word.word}-${index}`}
+          className={`${
+            isWordActive(word, currentTime) ? 'bg-yellow-200' : ''
+          } inline-block mr-1 cursor-pointer hover:bg-gray-100`}
+          onClick={() => {
+            if (videoRef.current) {
+              videoRef.current.currentTime = word.start;
+            }
+          }}
+        >
+          {word.word}
+        </span>
+      ))}
+    </div>
+  );
+};
+
+export default Transcripts;

packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/clips/Clip.tsx

@@ -4,12 +4,11 @@ import { Card, CardContent } from '@/components/ui/card';
 import { fetchAsset } from '@/lib/services/sessionService';
 import { IExtendedSession } from '@/lib/types';
 import { formatDate } from '@/lib/utils/time';
-import { Asset } from 'livepeer/models/components';
 import { useRouter } from 'next/navigation';
 import { useEffect, useState } from 'react';
 import { ProcessingStatus } from 'streameth-new-server/src/interfaces/session.interface';
 import Preview from './Preview';
-
+import { Asset } from 'livepeer/models/components/asset';
 export default function Clip({ session }: { session: IExtendedSession }) {
   const { name, coverImage, assetId } = session;
   const [asset, setAsset] = useState<Asset | null>(null);

packages/app/app/studio/[organization]/(no-side-bar)/clips/[stageId]/sidebar/index.tsx

@@ -7,17 +7,24 @@ import CreateClipButton from '../topBar/CreateClipButton';
 import AddOrEditMarkerForm from './markers/AddOrEditMarkerForm';
 import { IExtendedSession } from '@/lib/types';
 import ImportMarkersForm from './markers/ImportMarkersForm';
+import Transcripts from './Transcipts';
 
 export default function Sidebar({
   organizationId,
   stageSessions,
   liveRecordingId,
   animations,
+  words,
 }: {
   organizationId: string;
   stageSessions: IExtendedSession[];
   liveRecordingId?: string;
   animations: IExtendedSession[];
+  words?: {
+    word: string;
+    start: number;
+    end: number;
+  }[];
 }) {
   const { isCreatingClip, isAddingOrEditingMarker, isImportingMarkers } =
     useClipContext();
@@ -57,16 +64,22 @@ export default function Sidebar({
           )
       )}
       <Tabs defaultValue="clips" className="flex flex-col h-full">
-        <TabsList className="grid w-full grid-cols-2 flex-shrink-0">
+        <TabsList className="grid w-full grid-cols-3 flex-shrink-0">
           <TabsTrigger value="markers">Markers</TabsTrigger>
           <TabsTrigger value="clips">Clips</TabsTrigger>
+          {words && <TabsTrigger value="words">Words</TabsTrigger>}
         </TabsList>
         <TabsContent value="markers" className="flex-grow overflow-hidden">
           {<Markers organizationId={organizationId} />}
         </TabsContent>
         <TabsContent value="clips" className="flex-grow overflow-hidden">
           <SessionSidebar sessions={stageSessions} />
         </TabsContent>
+        {words && (
+          <TabsContent value="words" className="flex-grow overflow-auto p-4">
+            <Transcripts words={words} />
+          </TabsContent>
+        )}
       </Tabs>
     </div>
   );

packages/app/app/studio/[organization]/(root)/library/[session]/components/SessionTranscriptions.tsx

@@ -44,27 +44,41 @@ const SessionTranscriptions = ({
       });
   };
 
-  if (transcriptionState === TranscriptionStatus.processing) {
-    return (
-      <div className="flex items-center">
-        <LuLoader2 className="mr-2 w-4 h-4 animate-spin" /> Processing
-        transcription...{' '}
-        <p
-          className="pl-2 cursor-pointer"
-          title="refresh"
-          onClick={() => router.refresh()}
-        >
-          <LuRefreshCcw />
-        </p>
-      </div>
-    );
-  }
+  // if (transcriptionState === TranscriptionStatus.processing) {
+  //   return (
+  //     <div className="flex items-center">
+  //       <LuLoader2 className="mr-2 w-4 h-4 animate-spin" /> Processing
+  //       transcription...{' '}
+  //       <p
+  //         className="pl-2 cursor-pointer"
+  //         title="refresh"
+  //         onClick={() => router.refresh()}
+  //       >
+  //         <LuRefreshCcw />
+  //       </p>
+  //     </div>
+  //   );
+  // }
 
   if (
     transcriptionState === TranscriptionStatus.completed &&
     videoTranscription
   ) {
-    return <TextPlaceholder text={videoTranscription} />;
+    return (
+      <div className="space-y-4">
+        <TextPlaceholder text={videoTranscription} />
+        <Button
+          variant="outline"
+          size="sm"
+          onClick={handleGenerateTranscription}
+          loading={isGeneratingTranscript}
+          className="flex items-center gap-2"
+        >
+          <LuRefreshCcw className="w-4 h-4" />
+          Regenerate Transcription
+        </Button>
+      </div>
+    );
   }
 
   if (transcriptionState === TranscriptionStatus.failed) {

packages/app/app/studio/[organization]/(root)/library/[session]/page.tsx

@@ -29,7 +29,6 @@ const EditSession = async ({ params, searchParams }: studioPageParams) => {
     session: params.session,
   });
 
-  console.log(session?.transcripts?.chunks[0]);
   if (!session?.playbackId || !organization) return notFound();
 
   return (

packages/app/app/studio/[organization]/(root)/library/components/TableCells.tsx

@@ -106,7 +106,7 @@ const TableCells = async ({
       </TableCell>
       <TableCell className={rowBackgroundClass}>
         <div className="flex justify-end items-center space-x-2 max-w-[100px]">
-          {!isDisabled && item.type === 'livestream' && (
+          {!isDisabled && (
             // item.createdAt &&
             // new Date(item.createdAt).getTime() >
             //   Date.now() - 7 * 24 * 60 * 60 * 1000 && (

packages/app/lib/actions/livepeer.ts

@@ -22,6 +22,9 @@ export const getVideoUrlAction = async (
   session: IExtendedSession
 ): Promise<string | null> => {
   try {
+    if (session.playback?.videoUrl) {
+      return session.playback.videoUrl;
+    }
     if (session.assetId) {
       const asset = await fetchAsset({ assetId: session.assetId });
       if (asset?.playbackUrl) {

packages/app/lib/actions/sessions.ts

@@ -14,6 +14,7 @@ import {
   saveSessionImport,
   generateTranscriptions,
   uploadSessionToSocialsRequest,
+  extractHighlights,
 } from '../services/sessionService';
 import {
   ISession,
@@ -280,3 +281,13 @@ export const generateTranscriptionActions = async ({
     return null;
   }
 };
+
+
+export const extractHighlightsAction = async ({
+  sessionId,
+}: {
+  sessionId: string;
+}) => {
+  const res = await extractHighlights({ sessionId });
+  return res;
+};

packages/app/lib/services/sessionService.ts

@@ -7,7 +7,7 @@ import {
 import { apiUrl } from '@/lib/utils/utils';
 import { ISession } from 'streameth-new-server/src/interfaces/session.interface';
 import { revalidatePath } from 'next/cache';
-import { Asset } from 'livepeer/models/components';
+import { Asset } from 'livepeer/models/components/asset';
 import FuzzySearch from 'fuzzy-search';
 import { fetchClient } from './fetch-client';
 
@@ -563,3 +563,18 @@ export const generateTranscriptions = async ({
     throw e;
   }
 };
+
+export const extractHighlights = async ({
+  sessionId,
+}: {
+  sessionId: string;
+}) => {
+  try {
+    const response = await fetchClient(`${apiUrl()}/sessions/${sessionId}/highlights`, {
+      method: 'POST',
+    });
+  } catch (e) {
+    console.log('error in extractHighlights', e);
+    throw e;
+  }
+};

packages/server/package.json

@@ -40,6 +40,7 @@
     "fuse.js": "^7.0.0",
     "google-auth-library": "^9.6.3",
     "googleapis": "^133.0.0",
+    "gpt-3-encoder": "^1.1.4",
     "helmet": "^7.1.0",
     "hpp": "^0.2.3",
     "http-errors": "^2.0.0",

packages/server/src/controllers/session.controller.ts

@@ -199,4 +199,18 @@ export class SessionController extends Controller {
     await this.sessionService.deleteOne(sessionId);
     return SendApiResponse('deleted');
   }
+
+  /**
+ * @summary Extract highlights from session
+   */
+  @SuccessResponse('200')
+  @Post('{sessionId}/highlights')
+  async extractHighlights(
+    @Path() sessionId: string,
+  ): Promise<IStandardResponse<void>> {
+    const highlights = await this.sessionService.extractHighlights(sessionId);
+    return SendApiResponse(highlights.content);
+  }
 }
+
+

packages/server/src/databases/index.ts

@@ -8,9 +8,9 @@ console.log('Database:', name);
 console.log('Password length:', password?.length);
 
 export const dbConnection = {
-  url: `mongodb://${user}:${password}@${host}/${name}?authSource=admin&retryWrites=true&w=majority`,
+  //rl: `mongodb://${user}:${password}@${host}/${name}?authSource=admin&retryWrites=true&w=majority`,
   // For local development use this url
-  // url: `mongodb+srv://${user}:${password}@${host}/${name}?authSource=admin`,
+  url: `mongodb+srv://${user}:${password}@${host}/${name}?authSource=admin`,
   options: {
     useNewUrlParser: true,
     useUnifiedTopology: true,