Merge pull request #4452 from traPtitech/feat/livekit-sdk-init

カメラが使えるように

package.json

@@ -28,6 +28,7 @@
   "dependencies": {
     "@mdi/js": "^7.4.47",
     "@sapphi-red/web-noise-suppressor": "^0.3.5",
+    "@shiguredo/virtual-background": "^2023.2.0",
     "@traptitech/traq": "^3.17.0-3",
     "@traptitech/traq-markdown-it": "^6.3.0",
     "autosize": "^6.0.1",

src/components/Main/MainView/QallView/AudioTrack.vue

@@ -26,6 +26,7 @@ onUnmounted(() => {
 </script>
 
 <template>
+  <div>{{ trackInfo.participantIdentity }}</div>
   <audio :id="trackInfo.trackPublication?.trackSid" ref="audioElement"></audio>
   <input v-model="volume" type="range" min="0" max="1" step="0.01" />
 </template>

src/components/Main/MainView/QallView/QallView.vue

@@ -2,14 +2,39 @@
 import { useQall } from '/@/composables/qall/useQall'
 import VideoComponent from '/@/components/Main/MainView/QallView/VideoTrack.vue'
 import AudioComponent from '/@/components/Main/MainView/QallView/AudioTrack.vue'
+import { onMounted, ref } from 'vue'
 
-const { tracksMap, addScreenShareTrack } = useQall()
+const { tracksMap, addScreenShareTrack, addCameraTrack } = useQall()
+
+const videoInputs = ref<MediaDeviceInfo[]>([])
+onMounted(async () => {
+  const devices = await navigator.mediaDevices.enumerateDevices()
+  videoInputs.value = devices.filter(d => d.kind === 'videoinput')
+})
+const selectedVideoInput = ref<MediaDeviceInfo>()
 </script>
 
 <template>
   <div :class="$style.Block">
     <h1 :class="$style.Header">Qall View</h1>
     <button @click="addScreenShareTrack">Add Screen Share Track</button>
+    <select v-model="selectedVideoInput">
+      <option
+        v-for="videoInput in videoInputs"
+        :key="videoInput.deviceId"
+        :value="videoInput"
+      >
+        {{ videoInput.label }}
+      </option>
+    </select>
+    <button
+      @click="[
+        addCameraTrack(selectedVideoInput),
+        console.log(selectedVideoInput)
+      ]"
+    >
+      Add Camera Track
+    </button>
     <div :class="$style.TrackContainer">
       <template
         v-for="track of tracksMap.values()"

src/components/Main/MainView/QallView/VideoTrack.vue

@@ -8,7 +8,7 @@ const { trackInfo, participantIdentity } = defineProps<{
   participantIdentity: string
 }>()
 
-const { removeScreenShareTrack } = useQall()
+const { removeVideoTrack } = useQall()
 
 const videoElement = useTemplateRef<HTMLVideoElement>('videoElement')
 const volume = ref(1)
@@ -43,7 +43,7 @@ onUnmounted(() => {
     <input v-model="volume" type="range" min="0" max="1" step="0.01" />
     <button
       v-if="!trackInfo.isRemote && trackInfo.trackPublication"
-      @click="removeScreenShareTrack(trackInfo.trackPublication)"
+      @click="removeVideoTrack(trackInfo.trackPublication)"
     >
       Remove Screen Share
     </button>

src/composables/qall/useLiveKitSDK.ts

@@ -3,7 +3,9 @@ import {
   RoomEvent,
   AudioPresets,
   createLocalScreenTracks,
-  Room
+  Room,
+  createLocalVideoTrack,
+  LocalVideoTrack
 } from 'livekit-client'
 import type {
   RemoteTrack,
@@ -14,9 +16,13 @@ import type {
   Participant,
   LocalTrack
 } from 'livekit-client'
-import { ref, type Ref } from 'vue'
+import { ref, watch, type Ref } from 'vue'
 import { useToastStore } from '/@/store/ui/toast'
 import apis from '/@/lib/apis'
+import { VirtualBackgroundProcessor } from '@shiguredo/virtual-background'
+
+const virtualBackgroundAssetsPath =
+  'https://cdn.jsdelivr.net/npm/@shiguredo/virtual-background@latest/dist'
 
 const { addErrorToast } = useToastStore()
 
@@ -33,9 +39,15 @@ export type TrackInfo = (
   participantIdentity: string
 }
 
+type CameraProcessor = {
+  processor: VirtualBackgroundProcessor
+  track: MediaStreamVideoTrack
+}
+
 const room = ref<Room>()
 const speakerIdentity = ref<string[]>([])
 const tracksMap: Ref<Map<string, TrackInfo>> = ref(new Map())
+const cameraProcessorMap: Ref<Map<string, CameraProcessor>> = ref(new Map())
 
 function handleTrackSubscribed(
   track: RemoteTrack,
@@ -168,26 +180,86 @@ async function leaveRoom() {
 
 const Attributes = ref<{ [key: string]: string }>({})
 
+const addCameraTrack = async (
+  videoInputDevice?: MediaDeviceInfo,
+  isBlur?: boolean
+) => {
+  try {
+    if (!room.value) {
+      addErrorToast('ルームが存在しません')
+      return
+    }
+
+    const stream = await navigator.mediaDevices.getUserMedia({
+      video: { deviceId: videoInputDevice?.deviceId }
+    })
+    const track = stream.getVideoTracks()[0]
+    if (!track) {
+      addErrorToast('映像が取得できませんでした')
+      return
+    }
+    const processor = new VirtualBackgroundProcessor(
+      virtualBackgroundAssetsPath
+    )
+
+    const options = {
+      blurRadius: 15 // 背景ぼかし設定
+    }
+    const processedTrack = await processor.startProcessing(track, options)
+    const localTrack = new LocalVideoTrack(processedTrack)
+    await room.value?.localParticipant.publishTrack(localTrack).catch(e => {
+      addErrorToast('カメラの共有に失敗しました')
+      track.stop()
+      processor.stopProcessing()
+      return
+    })
+    if (localTrack.sid) {
+      console.log(localTrack.sid)
+      cameraProcessorMap.value.set(localTrack.sid, {
+        processor,
+        track
+      })
+    }
+  } catch {
+    addErrorToast('カメラの共有に失敗しました')
+    return
+  }
+}
+
 const addScreenShareTrack = async () => {
   try {
     if (!room.value) {
       addErrorToast('ルームが存在しません')
       return
     }
     const localTracks = await createLocalScreenTracks({
-      audio: true
+      audio: {
+        channelCount: 2,
+        autoGainControl: false,
+        noiseSuppression: false,
+        echoCancellation: false,
+        voiceIsolation: false
+      }
     })
 
     await Promise.all(
       localTracks.map(async t => {
-        await room.value?.localParticipant.publishTrack(t)
+        if (t.kind === Track.Kind.Video) {
+          await room.value?.localParticipant.publishTrack(t)
+        } else {
+          await room.value?.localParticipant.publishTrack(t, {
+            audioPreset: AudioPresets.musicHighQualityStereo,
+            dtx: false,
+            red: false
+          })
+        }
       })
     )
     const videoSid = localTracks.find(t => t.kind === Track.Kind.Video)?.sid
     const audioSid = localTracks.find(t => t.kind === Track.Kind.Audio)?.sid
     if (audioSid && videoSid) {
       Attributes.value = {
-        ...room.value.localParticipant.attributes,
+        ...Attributes.value,
         [videoSid]: audioSid
       }
       await room.value.localParticipant.setAttributes({
@@ -200,24 +272,29 @@ const addScreenShareTrack = async () => {
   }
 }
 
-const removeScreenShareTrack = async (
-  localpublication: LocalTrackPublication
-) => {
+const removeVideoTrack = async (localpublication: LocalTrackPublication) => {
   if (localpublication.track) {
     if (!room.value) {
       addErrorToast('ルームが存在しません')
       return
     }
+    const cameraProcessor = cameraProcessorMap.value.get(
+      localpublication.trackSid
+    )
+    if (cameraProcessor) {
+      console.log('stopProcessing')
+      cameraProcessor.processor.stopProcessing()
+      cameraProcessor.track.stop()
+      cameraProcessorMap.value.delete(localpublication.trackSid)
+    }
 
     const { [localpublication.trackSid]: audioSid, ...newAttributes } =
       Attributes.value
     //room.value.localParticipant.attributes
-    console.log(audioSid)
     await room.value.localParticipant.unpublishTrack(
       localpublication.track,
       true
     )
-    console.log(audioSid)
     room.value.localParticipant.setAttributes(newAttributes)
     Attributes.value = newAttributes
     if (!audioSid) {
@@ -262,7 +339,8 @@ export const useLiveKitSDK = () => {
     joinRoom,
     leaveRoom,
     addScreenShareTrack,
-    removeScreenShareTrack,
+    addCameraTrack,
+    removeVideoTrack,
     setTrackEnabled,
     setLocalTrackMute,
     tracksMap

src/composables/qall/useQall.ts

@@ -9,7 +9,8 @@ const {
   joinRoom,
   leaveRoom,
   addScreenShareTrack,
-  removeScreenShareTrack,
+  removeVideoTrack,
+  addCameraTrack,
   setLocalTrackMute,
   tracksMap
 } = useLiveKitSDK()
@@ -40,7 +41,8 @@ export const useQall = () => {
     isCalling,
     toggleCalling,
     addScreenShareTrack,
-    removeScreenShareTrack,
+    addCameraTrack,
+    removeVideoTrack,
     tracksMap
   }
 }