Add startVideo method and RoomEvent.VideoPlaybackStatusChanged (#939

)

* detect video playback issues

* Use suspend event

* revert attachToElm

* cleanup

* Create tasty-bears-float.md

* don't reset if a single play succeeds

* prettier

.changeset/tasty-bears-float.md

@@ -0,0 +1,5 @@
+---
+"livekit-client": patch
+---
+
+Add `startVideo` method and `RoomEvent.VideoPlaybackStatusChanged`

src/room/Room.ts

@@ -152,6 +152,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
   private regionUrl?: string;
 
+  private isVideoPlaybackBlocked: boolean = false;
+
   /**
    * Creates a new Room, the primary construct for a LiveKit session.
    * @param options
@@ -865,13 +867,36 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   };
 
+  startVideo = async () => {
+    for (const p of this.participants.values()) {
+      p.videoTracks.forEach((tr) => {
+        tr.track?.attachedElements.forEach((el) => {
+          el.play().catch((e) => {
+            if (e.name === 'NotAllowedError') {
+              log.warn(
+                'Resuming video playback failed, make sure you call `startVideo` directly in a user gesture handler',
+              );
+            }
+          });
+        });
+      });
+    }
+  };
+
   /**
    * Returns true if audio playback is enabled
    */
   get canPlaybackAudio(): boolean {
     return this.audioEnabled;
   }
 
+  /**
+   * Returns true if video playback is enabled
+   */
+  get canPlaybackVideo(): boolean {
+    return !this.isVideoPlaybackBlocked;
+  }
+
   /**
    * Returns the active audio output device used in this room.
    * @return the previously successfully set audio output device ID or an empty string if the default device is used.
@@ -1384,6 +1409,20 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     this.emit(RoomEvent.AudioPlaybackStatusChanged, false);
   };
 
+  private handleVideoPlaybackStarted = () => {
+    if (this.isVideoPlaybackBlocked) {
+      this.isVideoPlaybackBlocked = false;
+      this.emit(RoomEvent.VideoPlaybackStatusChanged, true);
+    }
+  };
+
+  private handleVideoPlaybackFailed = () => {
+    if (!this.isVideoPlaybackBlocked) {
+      this.isVideoPlaybackBlocked = true;
+      this.emit(RoomEvent.VideoPlaybackStatusChanged, false);
+    }
+  };
+
   private handleDeviceChange = async () => {
     this.emit(RoomEvent.MediaDevicesChanged);
   };
@@ -1487,6 +1526,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           if (track.kind === Track.Kind.Audio) {
             track.on(TrackEvent.AudioPlaybackStarted, this.handleAudioPlaybackStarted);
             track.on(TrackEvent.AudioPlaybackFailed, this.handleAudioPlaybackFailed);
+          } else if (track.kind === Track.Kind.Video) {
+            track.on(TrackEvent.VideoPlaybackFailed, this.handleVideoPlaybackFailed);
+            track.on(TrackEvent.VideoPlaybackStarted, this.handleVideoPlaybackStarted);
           }
           this.emit(RoomEvent.TrackSubscribed, track, publication, participant);
         },
@@ -1929,6 +1971,7 @@ export type RoomEventCallbacks = {
     participant: RemoteParticipant,
   ) => void;
   audioPlaybackChanged: (playing: boolean) => void;
+  videoPlaybackChanged: (playing: boolean) => void;
   signalConnected: () => void;
   recordingStatusChanged: (recording: boolean) => void;
   participantEncryptionStatusChanged: (encrypted: boolean, participant?: Participant) => void;

src/room/events.ts

@@ -245,6 +245,13 @@ export enum RoomEvent {
    */
   AudioPlaybackStatusChanged = 'audioPlaybackChanged',
 
+  /**
+   * LiveKit will attempt to autoplay all video tracks when you attach them to
+   * a video element. However, if that fails, we'll notify you via VideoPlaybackStatusChanged.
+   * Calling `room.startVideo()` in a user gesture event handler will resume the video playback.
+   */
+  VideoPlaybackStatusChanged = 'videoPlaybackChanged',
+
   /**
    * When we have encountered an error while attempting to create a track.
    * The errors take place in getUserMedia().
@@ -510,6 +517,10 @@ export enum TrackEvent {
   /** @internal */
   VideoDimensionsChanged = 'videoDimensionsChanged',
   /** @internal */
+  VideoPlaybackStarted = 'videoPlaybackStarted',
+  /** @internal */
+  VideoPlaybackFailed = 'videoPlaybackFailed',
+  /** @internal */
   ElementAttached = 'elementAttached',
   /** @internal */
   ElementDetached = 'elementDetached',

src/room/track/Track.ts

@@ -1,7 +1,7 @@
 import { EventEmitter } from 'events';
+import { debounce } from 'ts-debounce';
 import type TypedEventEmitter from 'typed-emitter';
 import type { SignalClient } from '../../api/SignalClient';
-import log from '../../logger';
 import { TrackSource, TrackType } from '../../proto/livekit_models_pb';
 import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc_pb';
 import { TrackEvent } from '../events';
@@ -113,6 +113,9 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
     if (!this.attachedElements.includes(element)) {
       this.attachedElements.push(element);
+      // listen to suspend events in order to detect auto playback issues
+      element.addEventListener('suspend', this.handleElementSuspended);
+      element.addEventListener('playing', this.handleElementPlay);
     }
 
     // even if we believe it's already attached to the element, it's possible
@@ -130,11 +133,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
           this.emit(TrackEvent.AudioPlaybackStarted);
         })
         .catch((e) => {
-          if (e.name === 'NotAllowedError') {
-            this.emit(TrackEvent.AudioPlaybackFailed, e);
-          } else {
-            log.warn('could not playback audio', e);
-          }
           // If audio playback isn't allowed make sure we still play back the video
           if (
             element &&
@@ -172,6 +170,8 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
         if (idx >= 0) {
           this.attachedElements.splice(idx, 1);
           this.recycleElement(element);
+          element.removeEventListener('suspend', this.handleElementSuspended);
+          element.removeEventListener('playing', this.handleElementPlay);
           this.emit(TrackEvent.ElementDetached, element);
         }
         return element;
@@ -182,6 +182,8 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
         detachTrack(this.mediaStreamTrack, elm);
         detached.push(elm);
         this.recycleElement(elm);
+        elm.removeEventListener('suspend', this.handleElementSuspended);
+        elm.removeEventListener('playing', this.handleElementPlay);
         this.emit(TrackEvent.ElementDetached, elm);
       });
 
@@ -268,9 +270,26 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
       document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
     }
   }
+
+  private handleElementSuspended = () => {
+    this.debouncedPlaybackStateChange(false);
+  };
+
+  private handleElementPlay = () => {
+    this.debouncedPlaybackStateChange(true);
+  };
+
+  private debouncedPlaybackStateChange = debounce((allowed: boolean) => {
+    // we debounce this as Safari triggers both `playing` and `suspend` shortly after one another
+    // in order not to raise the wrong event, we debounce the call to make sure we only emit the correct status
+    if (this.kind === Track.Kind.Audio) {
+      this.emit(allowed ? TrackEvent.AudioPlaybackStarted : TrackEvent.AudioPlaybackFailed);
+    } else if (this.kind === Track.Kind.Video) {
+      this.emit(allowed ? TrackEvent.VideoPlaybackStarted : TrackEvent.VideoPlaybackFailed);
+    }
+  }, 300);
 }
 
-/** @internal */
 export function attachToElement(track: MediaStreamTrack, element: HTMLMediaElement) {
   let mediaStream: MediaStream;
   if (element.srcObject instanceof MediaStream) {
@@ -321,7 +340,7 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
         // when the window is backgrounded before the first frame is drawn
         // manually calling play here seems to fix that
         element.play().catch(() => {
-          /* do nothing */
+          /** do nothing, we watch the `suspended` event do deal with these failures */
         });
       }, 0);
     }
@@ -446,10 +465,12 @@ export type TrackEventCallbacks = {
   updateSettings: () => void;
   updateSubscription: () => void;
   audioPlaybackStarted: () => void;
-  audioPlaybackFailed: (error: Error) => void;
+  audioPlaybackFailed: (error?: Error) => void;
   audioSilenceDetected: () => void;
   visibilityChanged: (visible: boolean, track?: any) => void;
   videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void;
+  videoPlaybackStarted: () => void;
+  videoPlaybackFailed: (error?: Error) => void;
   elementAttached: (element: HTMLMediaElement) => void;
   elementDetached: (element: HTMLMediaElement) => void;
   upstreamPaused: (track: any) => void;