From 7de84c876fb8c0b8965d64a83a9ee87d9ba9be31 Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Mon, 18 Nov 2024 16:06:44 -0500 Subject: [PATCH 1/6] ref(TPC) Move all the utility functions to TPCUtils. --- modules/RTC/TPCUtils.js | 572 ++++++++++------- modules/RTC/TPCUtils.spec.js | 117 ++-- modules/RTC/TraceablePeerConnection.js | 625 ++++++++----------- modules/xmpp/JingleSessionPC.js | 2 +- types/hand-crafted/modules/RTC/TPCUtils.d.ts | 1 - 5 files changed, 630 insertions(+), 687 deletions(-) diff --git a/modules/RTC/TPCUtils.js b/modules/RTC/TPCUtils.js index ada4e85a23..e6922809c5 100644 --- a/modules/RTC/TPCUtils.js +++ b/modules/RTC/TPCUtils.js @@ -5,7 +5,6 @@ import transform from 'sdp-transform'; import { CodecMimeType } from '../../service/RTC/CodecMimeType'; import { MediaDirection } from '../../service/RTC/MediaDirection'; import { MediaType } from '../../service/RTC/MediaType'; -import { getSourceIndexFromSourceName } from '../../service/RTC/SignalingLayer'; import { SIM_LAYERS, STANDARD_CODEC_SETTINGS, @@ -15,13 +14,17 @@ import { import { VideoEncoderScalabilityMode } from '../../service/RTC/VideoEncoderScalabilityMode'; import { VideoType } from '../../service/RTC/VideoType'; import browser from '../browser'; +import SDPUtil from '../sdp/SDPUtil'; const logger = getLogger(__filename); +const DD_HEADER_EXT_URI + = 'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension'; +const DD_HEADER_EXT_ID = 11; const VIDEO_CODECS = [ CodecMimeType.AV1, CodecMimeType.H264, CodecMimeType.VP8, CodecMimeType.VP9 ]; /** - * Handles track related operations on TraceablePeerConnection when browser is - * running in unified plan mode. + * Handles all the utility functions for the TraceablePeerConnection class, like calculating the encoding parameters, + * determining the media direction, calculating bitrates based on the current codec settings, etc. */ export class TPCUtils { /** @@ -29,10 +32,22 @@ export class TPCUtils { * * @param peerconnection - the tpc instance for which we have utility functions. */ - constructor(peerconnection) { + constructor(peerconnection, options) { this.pc = peerconnection; + this.options = options; this.codecSettings = cloneDeep(STANDARD_CODEC_SETTINGS); - const videoQualitySettings = this.pc.options?.videoQuality; + + /** + * Flag indicating bridge support for AV1 codec. On the bridge connection, it is supported only when support for + * Dependency Descriptor header extensions is offered by Jicofo. H.264 simulcast is also possible when these + * header extensions are negotiated. + */ + this.supportsDDHeaderExt = false; + + /** + * Reads videoQuality settings from config.js and overrides the code defaults for video codecs. + */ + const videoQualitySettings = this.options.videoQuality; if (videoQualitySettings) { for (const codec of VIDEO_CODECS) { @@ -86,7 +101,7 @@ export class TPCUtils { const codecBitrates = this.codecSettings[codec].maxBitratesVideo; const trackCaptureHeight = localVideoTrack.getCaptureResolution(); const effectiveNewHeight = newHeight > trackCaptureHeight ? trackCaptureHeight : newHeight; - const desktopShareBitrate = this.pc.options?.videoQuality?.desktopbitrate || codecBitrates.ssHigh; + const desktopShareBitrate = this.options.videoQuality?.desktopbitrate || codecBitrates.ssHigh; const isScreenshare = localVideoTrack.getVideoType() === VideoType.DESKTOP; let scalabilityMode = this.codecSettings[codec].useKSVC ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3; @@ -140,82 +155,17 @@ export class TPCUtils { } /** - * Configures the RTCRtpEncodingParameters of the outbound rtp stream associated with the given track. - * - * @param {JitsiLocalTracj} localTrack - The local track whose outbound stream needs to be configured. - * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise. - */ - _configureSenderEncodings(localTrack) { - const mediaType = localTrack.getType(); - const transceiver = localTrack?.track && localTrack.getOriginalStream() - ? this.pc.peerconnection.getTransceivers().find(t => t.sender?.track?.id === localTrack.getTrackId()) - : this.pc.peerconnection.getTransceivers().find(t => t.receiver?.track?.kind === mediaType); - const parameters = transceiver?.sender?.getParameters(); - - // Resolve if the encodings are not available yet. This happens immediately after the track is added to the - // peerconnection on chrome in unified-plan. It is ok to ignore and not report the error here since the - // action that triggers 'addTrack' (like unmute) will also configure the encodings and set bitrates after that. - if (!parameters?.encodings?.length) { - return Promise.resolve(); - } - - parameters.encodings = this._getStreamEncodings(localTrack); - - return transceiver.sender.setParameters(parameters); - } - - /** - * Enables/disables the streams by changing the active field on RTCRtpEncodingParameters for a given RTCRtpSender. - * - * @param {RTCRtpSender} sender - the sender associated with a MediaStreamTrack. - * @param {boolean} enable - whether the streams needs to be enabled or disabled. - * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise. - */ - _enableSenderEncodings(sender, enable) { - const parameters = sender.getParameters(); - - if (parameters?.encodings?.length) { - for (const encoding of parameters.encodings) { - encoding.active = enable; - } - } - - return sender.setParameters(parameters); - } - - /** - * Obtains stream encodings that need to be configured on the given track based - * on the track media type and the simulcast setting. - * @param {JitsiLocalTrack} localTrack - */ - _getStreamEncodings(localTrack) { - if (localTrack.isAudioTrack()) { - return [ { active: this.pc.audioTransferActive } ]; - } - const codec = this.pc.getConfiguredVideoCodec(localTrack); - - if (this.pc.isSpatialScalabilityOn()) { - return this._getVideoStreamEncodings(localTrack, codec); - } - - return [ { - active: this.pc.videoTransferActive, - maxBitrate: this.codecSettings[codec].maxBitratesVideo.high - } ]; - } - - /** - * The startup configuration for the stream encodings that are applicable to - * the video stream when a new sender is created on the peerconnection. The initial - * config takes into account the differences in browser's simulcast implementation. + * The startup configuration for the stream encodings that are applicable to the video stream when a new sender is + * created on the peerconnection. The initial config takes into account the differences in browser's simulcast + * implementation. * * Encoding parameters: * active - determine the on/off state of a particular encoding. - * maxBitrate - max. bitrate value to be applied to that particular encoding - * based on the encoding's resolution and config.js videoQuality settings if applicable. + * maxBitrate - max. bitrate value to be applied to that particular encoding based on the encoding's resolution and + * config.js videoQuality settings if applicable. * rid - Rtp Stream ID that is configured for a particular simulcast stream. - * scaleResolutionDownBy - the factor by which the encoding is scaled down from the - * original resolution of the captured video. + * scaleResolutionDownBy - the factor by which the encoding is scaled down from the original resolution of the + * captured video. * * @param {JitsiLocalTrack} localTrack * @param {String} codec @@ -339,68 +289,6 @@ export class TPCUtils { && !browser.isWebKitBased(); } - /** - * Updates the sender parameters in the stream encodings. - * - * @param {RTCRtpSender} sender - the sender associated with a MediaStreamTrack. - * @param {boolean} enable - whether the streams needs to be enabled or disabled. - * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise. - */ - _updateSenderEncodings(sender, enable) { - const parameters = sender.getParameters(); - - if (parameters?.encodings?.length) { - for (const encoding of parameters.encodings) { - encoding.active = enable; - } - } - - return sender.setParameters(parameters); - } - - /** - * Adds {@link JitsiLocalTrack} to the WebRTC peerconnection for the first time. - * - * @param {JitsiLocalTrack} track - track to be added to the peerconnection. - * @param {boolean} isInitiator - boolean that indicates if the endpoint is offerer in a p2p connection. - * @returns {RTCRtpTransceiver} - the transceiver that the track was added to. - */ - addTrack(localTrack, isInitiator) { - const track = localTrack.getTrack(); - let transceiver; - - if (isInitiator) { - const streams = []; - - if (localTrack.getOriginalStream()) { - streams.push(localTrack.getOriginalStream()); - } - - // Use pc.addTransceiver() for the initiator case when local tracks are getting added - // to the peerconnection before a session-initiate is sent over to the peer. - const transceiverInit = { - direction: MediaDirection.SENDRECV, - streams, - sendEncodings: [] - }; - - if (!browser.isFirefox()) { - transceiverInit.sendEncodings = this._getStreamEncodings(localTrack); - } - transceiver = this.pc.peerconnection.addTransceiver(track, transceiverInit); - } else { - // Use pc.addTrack() for responder case so that we can re-use the m-lines that were created - // when setRemoteDescription was called. pc.addTrack() automatically attaches to any existing - // unused "recv-only" transceiver. - const sender = this.pc.peerconnection.addTrack(track); - - // Find the corresponding transceiver that the track was attached to. - transceiver = this.pc.peerconnection.getTransceivers().find(t => t.sender === sender); - } - - return transceiver; - } - /** * Returns the calculated active state of the stream encodings based on the frame height requested for the send * stream. All the encodings that have a resolution lower than the frame height requested will be enabled. @@ -465,7 +353,7 @@ export class TPCUtils { */ calculateEncodingsBitrates(localVideoTrack, codec, newHeight) { const codecBitrates = this.codecSettings[codec].maxBitratesVideo; - const desktopShareBitrate = this.pc.options?.videoQuality?.desktopbitrate || codecBitrates.ssHigh; + const desktopShareBitrate = this.options.videoQuality?.desktopbitrate || codecBitrates.ssHigh; const encodingsBitrates = this._getVideoStreamEncodings(localVideoTrack, codec) .map((encoding, idx) => { let bitrate = encoding.maxBitrate; @@ -588,6 +476,100 @@ export class TPCUtils { }); } + /** + * Returns the codec that is configured on the client as the preferred video codec for the given local video track. + * + * @param {JitsiLocalTrack} localTrack - The local video track. + * @returns {CodecMimeType} The codec that is set as the preferred codec for the given local video track. + */ + getConfiguredVideoCodec(localTrack) { + const localVideoTrack = localTrack ?? this.pc.getLocalVideoTracks()[0]; + const rtpSender = this.pc.findSenderForTrack(localVideoTrack.getTrack()); + + if (this.pc.usesCodecSelectionAPI() && rtpSender) { + const { codecs } = rtpSender.getParameters(); + + return codecs[0].mimeType.split('/')[1].toLowerCase(); + } + + const sdp = this.pc.remoteDescription?.sdp; + const defaultCodec = CodecMimeType.VP8; + + if (!sdp) { + return defaultCodec; + } + const parsedSdp = transform.parse(sdp); + const mLine = parsedSdp.media + .find(m => m.mid.toString() === this.pc.localTrackTransceiverMids.get(localVideoTrack.rtcId)); + const payload = mLine.payloads.split(' ')[0]; + const { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload)); + + if (codec) { + return Object.values(CodecMimeType).find(value => value === codec.toLowerCase()); + } + + return defaultCodec; + } + + /** + * Returns the codecs in the current order of preference as configured on the peerconnection. + * + * @param {RTCSessionDescription} - The local description to be used. + * @returns {Array} + */ + getConfiguredVideoCodecs(description) { + const currentSdp = description?.sdp ?? this.pc.localDescription?.sdp; + + if (!currentSdp) { + return []; + } + const parsedSdp = transform.parse(currentSdp); + const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO); + const codecs = new Set(mLine.rtp + .filter(pt => pt.codec.toLowerCase() !== 'rtx') + .map(pt => pt.codec.toLowerCase())); + + return Array.from(codecs); + } + + /** + * Returns the desired media direction for the given media type based on the current state of the peerconnection. + * + * @param {MediaType} mediaType - The media type for which the desired media direction is to be obtained. + * @param {boolean} isAddOperation - Whether the direction is being set for a source add operation. + * @returns {MediaDirection} - The desired media direction for the given media type. + */ + getDesiredMediaDirection(mediaType, isAddOperation = false) { + const hasLocalSource = this.pc.getLocalTracks(mediaType).length > 0; + + if (isAddOperation) { + return hasLocalSource ? MediaDirection.SENDRECV : MediaDirection.SENDONLY; + } + + return hasLocalSource ? MediaDirection.RECVONLY : MediaDirection.INACTIVE; + } + + /** + * Obtains stream encodings that need to be configured on the given track based + * on the track media type and the simulcast setting. + * @param {JitsiLocalTrack} localTrack + */ + getStreamEncodings(localTrack) { + if (localTrack.isAudioTrack()) { + return [ { active: this.pc.audioTransferActive } ]; + } + const codec = this.getConfiguredVideoCodec(localTrack); + + if (this.pc.isSpatialScalabilityOn()) { + return this._getVideoStreamEncodings(localTrack, codec); + } + + return [ { + active: this.pc.videoTransferActive, + maxBitrate: this.codecSettings[codec].maxBitratesVideo.high + } ]; + } + /** * Takes in a *unified plan* offer and inserts the appropriate parameters for adding simulcast receive support. * @param {Object} desc - A session description object @@ -622,7 +604,7 @@ export class TPCUtils { const simulcastLine = `recv ${ridLine}`; const sdp = transform.parse(desc.sdp); const mLines = sdp.media.filter(m => m.type === MediaType.VIDEO); - const senderMids = Array.from(this.pc._localTrackTransceiverMids.values()); + const senderMids = Array.from(this.pc.localTrackTransceiverMids.values()); mLines.forEach((mLine, idx) => { // Make sure the simulcast recv line is only set on video descriptions that are associated with senders. @@ -671,122 +653,248 @@ export class TPCUtils { } /** - * Replaces the existing track on a RTCRtpSender with the given track. + * Munges the session description to ensure that the codec order is as per the preferred codec settings. * - * @param {JitsiLocalTrack} oldTrack - existing track on the sender that needs to be removed. - * @param {JitsiLocalTrack} newTrack - new track that needs to be added to the sender. - * @returns {Promise} - resolved with the associated transceiver when done, rejected otherwise. + * @param {RTCSessionDescription} description - the local/remote description to be munged. + * @returns {RTCSessionDescription} - the munged local/remote description. */ - replaceTrack(oldTrack, newTrack) { - const mediaType = newTrack?.getType() ?? oldTrack?.getType(); - const localTracks = this.pc.getLocalTracks(mediaType); - const track = newTrack?.getTrack() ?? null; - const isNewLocalSource = localTracks?.length - && !oldTrack - && newTrack - && !localTracks.find(t => t === newTrack); - let transceiver; - - // If old track exists, replace the track on the corresponding sender. - if (oldTrack && !oldTrack.isMuted()) { - transceiver = this.pc.peerconnection.getTransceivers().find(t => t.sender.track === oldTrack.getTrack()); - - // Find the first recvonly transceiver when more than one track of the same media type is being added to the pc. - // As part of the track addition, a new m-line was added to the remote description with direction set to - // recvonly. - } else if (isNewLocalSource) { - transceiver = this.pc.peerconnection.getTransceivers().find( - t => t.receiver.track.kind === mediaType - && t.direction === MediaDirection.RECVONLY - - // Re-use any existing recvonly transceiver (if available) for p2p case. - && ((this.pc.isP2P && t.currentDirection === MediaDirection.RECVONLY) - || (t.currentDirection === MediaDirection.INACTIVE && !t.stopped))); - - // For mute/unmute operations, find the transceiver based on the track index in the source name if present, - // otherwise it is assumed to be the first local track that was added to the peerconnection. - } else { - transceiver = this.pc.peerconnection.getTransceivers().find(t => t.receiver.track.kind === mediaType); - const sourceName = newTrack?.getSourceName() ?? oldTrack?.getSourceName(); - - if (sourceName) { - const trackIndex = getSourceIndexFromSourceName(sourceName); - - if (this.pc.isP2P) { - transceiver = this.pc.peerconnection.getTransceivers() - .filter(t => t.receiver.track.kind === mediaType)[trackIndex]; - } else if (oldTrack) { - const transceiverMid = this.pc._localTrackTransceiverMids.get(oldTrack.rtcId); - - transceiver = this.pc.peerconnection.getTransceivers().find(t => t.mid === transceiverMid); - } else if (trackIndex) { - transceiver = this.pc.peerconnection.getTransceivers() - .filter(t => t.receiver.track.kind === mediaType - && t.direction !== MediaDirection.RECVONLY)[trackIndex]; + mungeCodecOrder(description) { + const codecSettings = this.pc.codecSettings; + + if (!codecSettings) { + return description; + } + + const { isP2P } = this.options; + const parsedSdp = transform.parse(description.sdp); + const mLines = parsedSdp.media.filter(m => m.type === codecSettings.mediaType); + + if (!mLines.length) { + return description; + } + + for (const mLine of mLines) { + const currentCodecs = this.getConfiguredVideoCodecs(description); + + for (const codec of currentCodecs) { + if (isP2P) { + // 1. Strip the high profile H264 codecs on all clients. macOS started offering encoder for H.264 + // level 5.2 but a decoder only for level 3.1. Therfore, strip all main and high level codecs for + // H.264. + // 2. There are multiple VP9 payload types generated by the browser, more payload types are added + // if the endpoint doesn't have a local video source. Therefore, strip all the high profile codec + // variants for VP9 so that only one payload type for VP9 is negotiated between the peers. + if (codec === CodecMimeType.H264 || codec === CodecMimeType.VP9) { + SDPUtil.stripCodec(mLine, codec, true /* high profile */); + } + + // Do not negotiate ULPFEC and RED either. + if (codec === CodecMimeType.ULPFEC || codec === CodecMimeType.RED) { + SDPUtil.stripCodec(mLine, codec, false); + } + } + } + + // Reorder the codecs based on the preferred settings. + if (!this.pc.usesCodecSelectionAPI()) { + for (const codec of codecSettings.codecList.slice().reverse()) { + SDPUtil.preferCodec(mLine, codec, isP2P); } } } - if (!transceiver) { - return Promise.reject( - new Error(`Replace track failed - no transceiver for old: ${oldTrack}, new: ${newTrack}`)); - } - logger.debug(`${this.pc} Replacing ${oldTrack} with ${newTrack}`); - return transceiver.sender.replaceTrack(track) - .then(() => Promise.resolve(transceiver)); + return new RTCSessionDescription({ + type: description.type, + sdp: transform.write(parsedSdp) + }); } /** - * Set the simulcast stream encoding properties on the RTCRtpSender. + * Munges the stereo flag as well as the opusMaxAverageBitrate in the SDP, based on values set through config.js, + * if present. * - * @param {JitsiLocalTrack} localTrack - the current track in use for which the encodings are to be set. - * @returns {Promise} - resolved when done. + * @param {RTCSessionDescription} description that needs to be munged. + * @returns {RTCSessionDescription} the munged description. */ - setEncodings(localTrack) { - if (localTrack.getType() === MediaType.VIDEO) { - return this.pc._updateVideoSenderParameters(() => this._configureSenderEncodings(localTrack)); + mungeOpus(description) { + const { audioQuality } = this.options; + + if (!audioQuality?.enableOpusDtx && !audioQuality?.stereo && !audioQuality?.opusMaxAverageBitrate) { + return description; } - return this._configureSenderEncodings(localTrack); + const parsedSdp = transform.parse(description.sdp); + const mLines = parsedSdp.media.filter(m => m.type === MediaType.AUDIO); + + for (const mLine of mLines) { + const { payload } = mLine.rtp.find(protocol => protocol.codec === CodecMimeType.OPUS); + + if (!payload) { + // eslint-disable-next-line no-continue + continue; + } + + let fmtpOpus = mLine.fmtp.find(protocol => protocol.payload === payload); + + if (!fmtpOpus) { + fmtpOpus = { + payload, + config: '' + }; + } + + const fmtpConfig = transform.parseParams(fmtpOpus.config); + let sdpChanged = false; + + if (audioQuality?.stereo) { + fmtpConfig.stereo = 1; + sdpChanged = true; + } + + if (audioQuality?.opusMaxAverageBitrate) { + fmtpConfig.maxaveragebitrate = audioQuality.opusMaxAverageBitrate; + sdpChanged = true; + } + + // On Firefox, the OpusDtx enablement has no effect + if (!browser.isFirefox() && audioQuality?.enableOpusDtx) { + fmtpConfig.usedtx = 1; + sdpChanged = true; + } + + if (!sdpChanged) { + // eslint-disable-next-line no-continue + continue; + } + + let mungedConfig = ''; + + for (const key of Object.keys(fmtpConfig)) { + mungedConfig += `${key}=${fmtpConfig[key]}; `; + } + + fmtpOpus.config = mungedConfig.trim(); + } + + return new RTCSessionDescription({ + type: description.type, + sdp: transform.write(parsedSdp) + }); } /** - * Resumes or suspends media on the peerconnection by setting the active state on RTCRtpEncodingParameters - * associated with all the senders that have a track attached to it. + * Munges the session description by setting the max bitrates on the video m-lines when VP9 K-SVC codec is in use. * - * @param {boolean} enable - whether outgoing media needs to be enabled or disabled. - * @param {string} mediaType - media type, 'audio' or 'video', if neither is passed, all outgoing media will either - * be enabled or disabled. - * @returns {Promise} - A promise that is resolved when the change is succesful on all the senders, rejected - * otherwise. + * @param {RTCSessionDescription} description - The local/remote description that needs to be munged. + * @param {boolean} isLocalSdp - Whether the max bitrate (via b=AS line in SDP) is set on local SDP. + * @returns {RTCSessionDescription} - The munged local/remote description. */ - setMediaTransferActive(enable, mediaType) { - logger.info(`${this.pc} ${enable ? 'Resuming' : 'Suspending'} media transfer.`); + setMaxBitrates(description, isLocalSdp = false) { + const pcCodecSettings = this.pc.codecSettings; + + if (!pcCodecSettings) { + return description; + } + const parsedSdp = transform.parse(description.sdp); - const senders = this.pc.peerconnection.getSenders() - .filter(s => Boolean(s.track) && (!mediaType || s.track.kind === mediaType)); - const promises = []; + // Find all the m-lines associated with the local sources. + const direction = isLocalSdp ? MediaDirection.RECVONLY : MediaDirection.SENDONLY; + const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO && m.direction !== direction); + const currentCodec = pcCodecSettings.codecList[0]; + const codecScalabilityModeSettings = this.codecSettings[currentCodec]; - for (const sender of senders) { - if (sender.track.kind === MediaType.VIDEO) { - promises.push(this.pc._updateVideoSenderParameters(() => this._enableSenderEncodings(sender, enable))); + for (const mLine of mLines) { + const isDoingVp9KSvc = currentCodec === CodecMimeType.VP9 + && !codecScalabilityModeSettings.scalabilityModeEnabled; + const localTrack = this.pc.getLocalVideoTracks() + .find(track => this.pc.localTrackTransceiverMids.get(track.rtcId) === mLine.mid.toString()); + + if (localTrack + && (isDoingVp9KSvc + + // Setting bitrates in the SDP for SVC codecs is no longer needed in the newer versions where + // maxBitrates from the RTCRtpEncodingParameters directly affect the target bitrate for the encoder. + || (this._isRunningInFullSvcMode(currentCodec) && !this.pc.usesCodecSelectionAPI()))) { + let maxBitrate; + + if (localTrack.getVideoType() === VideoType.DESKTOP) { + maxBitrate = codecScalabilityModeSettings.maxBitratesVideo.ssHigh; + } else { + const { level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= localTrack.getCaptureResolution()); + + maxBitrate = codecScalabilityModeSettings.maxBitratesVideo[level]; + } + + const limit = Math.floor(maxBitrate / 1000); + + // Use only the highest spatial layer bitrates for now as there is no API available yet for configuring + // the bitrates on the individual SVC layers. + mLine.bandwidth = [ { + type: 'AS', + limit + } ]; } else { - promises.push(this._enableSenderEncodings(sender, enable)); + // Clear the bandwidth limit in SDP when VP9 is no longer the preferred codec. + // This is needed on react native clients as react-native-webrtc returns the + // SDP that the application passed instead of returning the SDP off the native side. + // This line automatically gets cleared on web on every renegotiation. + mLine.bandwidth = undefined; } } - return Promise.allSettled(promises) - .then(settledResult => { - const errors = settledResult - .filter(result => result.status === 'rejected') - .map(result => result.reason); + return new RTCSessionDescription({ + type: description.type, + sdp: transform.write(parsedSdp) + }); + } + + /** + * Checks if the AV1 Dependency descriptors are negotiated on the bridge peerconnection and removes them from the + * description when codec selected is VP8 or VP9. + * + * @param {RTCSessionDescription} description that needs to be munged. + * @returns {RTCSessionDescription} the munged description. + */ + updateAv1DdHeaders(description) { + const parsedSdp = transform.parse(description.sdp); + const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO); + + if (!mLines.length || !browser.supportsDDExtHeaders()) { + return description; + } + + mLines.forEach((mLine, idx) => { + const senderMids = Array.from(this.pc.localTrackTransceiverMids.values()); + const isSender = senderMids.length + ? senderMids.find(mid => mLine.mid.toString() === mid.toString()) + : idx === 0; + const payload = mLine.payloads.split(' ')[0]; + let { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload)); + + codec = codec.toLowerCase(); - if (errors.length) { - return Promise.reject(new Error('Failed to change encodings on the RTCRtpSenders' - + `${errors.join(' ')}`)); + if (isSender && mLine.ext?.length) { + const headerIndex = mLine.ext.findIndex(ext => ext.uri === DD_HEADER_EXT_URI); + const shouldNegotiateHeaderExts = codec === CodecMimeType.AV1 || codec === CodecMimeType.H264; + + if (!this.supportsDDHeaderExt && headerIndex >= 0) { + this.supportsDDHeaderExt = true; } - return Promise.resolve(); - }); + if (this.supportsDDHeaderExt && shouldNegotiateHeaderExts && headerIndex < 0) { + mLine.ext.push({ + value: DD_HEADER_EXT_ID, + uri: DD_HEADER_EXT_URI + }); + } else if (!shouldNegotiateHeaderExts && headerIndex >= 0) { + mLine.ext.splice(headerIndex, 1); + } + } + }); + + return new RTCSessionDescription({ + type: description.type, + sdp: transform.write(parsedSdp) + }); } } diff --git a/modules/RTC/TPCUtils.spec.js b/modules/RTC/TPCUtils.spec.js index 9759ba0e06..9faf4d5b6c 100644 --- a/modules/RTC/TPCUtils.spec.js +++ b/modules/RTC/TPCUtils.spec.js @@ -35,7 +35,7 @@ describe('TPCUtils', () => { it('sort ssrcs associated with all FID ssrc-groups', () => { const pc = new MockPeerConnection(); - const tpcUtils = new TPCUtils(pc); + const tpcUtils = new TPCUtils(pc, { }); const source = new RTCSessionDescription({ type: 'offer', sdp: getSourceSdp() @@ -99,7 +99,7 @@ describe('TPCUtils', () => { it('sort ssrcs in case the first ssrc in the SIM group is not present at the top', () => { const pc = new MockPeerConnection(); - const tpcUtils = new TPCUtils(pc); + const tpcUtils = new TPCUtils(pc, { }); const source = new RTCSessionDescription({ type: 'offer', sdp: getSourceSdp() @@ -163,7 +163,7 @@ describe('TPCUtils', () => { it('sort ssrcs in case there is a single FID group', () => { const pc = new MockPeerConnection(); - const tpcUtils = new TPCUtils(pc); + const tpcUtils = new TPCUtils(pc, { }); const source = new RTCSessionDescription({ type: 'offer', sdp: getSourceSdp() @@ -223,9 +223,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -311,9 +310,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -442,9 +440,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = true; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -489,9 +486,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); pc._capScreenshareBitrate = true; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -531,9 +527,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = false; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -577,9 +572,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -660,9 +654,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -698,9 +691,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -787,9 +779,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = true; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -834,9 +825,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = false; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -880,9 +870,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -989,9 +978,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1098,9 +1086,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1166,9 +1153,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1204,9 +1190,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1299,9 +1284,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = true; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1348,9 +1332,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = false; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1396,9 +1379,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1464,9 +1446,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1502,9 +1483,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1614,9 +1594,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = true; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1661,9 +1640,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); pc._capScreenshareBitrate = false; - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1707,9 +1685,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1775,9 +1752,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1843,10 +1819,9 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc._capScreenshareBitrate = false; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -1908,9 +1883,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2034,9 +2008,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2154,9 +2127,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2266,9 +2238,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2378,10 +2349,9 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc._capScreenshareBitrate = true; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2447,10 +2417,9 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc._capScreenshareBitrate = false; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2516,9 +2485,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, false /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2602,9 +2570,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2689,9 +2656,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc.videoTransferActive = true; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality }); }); afterEach(() => { @@ -2776,10 +2742,9 @@ describe('TPCUtils', () => { }; pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc._capScreenshareBitrate = true; pc.videoTransferActive = true; - const utils = new TPCUtils(pc); + const utils = new TPCUtils(pc, { videoQuality }); it('and requested desktop resolution is 2160', () => { height = 2160; @@ -2894,10 +2859,9 @@ describe('TPCUtils', () => { const videoQuality = {}; pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = { videoQuality }; pc._capScreenshareBitrate = true; pc.videoTransferActive = true; - const utils = new TPCUtils(pc); + const utils = new TPCUtils(pc, { videoQuality }); it('and requested desktop resolution is 2160', () => { height = 2160; @@ -3003,9 +2967,8 @@ describe('TPCUtils', () => { beforeEach(() => { pc = new MockPeerConnection('1', true, true /* simulcast */); - pc.options = {}; pc.videoTransferActive = false; - tpcUtils = new TPCUtils(pc); + tpcUtils = new TPCUtils(pc, { videoQuality: {} }); }); afterEach(() => { diff --git a/modules/RTC/TraceablePeerConnection.js b/modules/RTC/TraceablePeerConnection.js index c0cddecb8e..12ee794642 100644 --- a/modules/RTC/TraceablePeerConnection.js +++ b/modules/RTC/TraceablePeerConnection.js @@ -31,9 +31,6 @@ import { TPCUtils } from './TPCUtils'; const logger = getLogger(__filename); const DEGRADATION_PREFERENCE_CAMERA = 'maintain-framerate'; const DEGRADATION_PREFERENCE_DESKTOP = 'maintain-resolution'; -const DD_HEADER_EXT_URI - = 'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension'; -const DD_HEADER_EXT_ID = 11; /* eslint-disable max-params */ @@ -245,7 +242,11 @@ export default function TraceablePeerConnection( this.peerconnection = new RTCPeerConnection(pcConfig, safeConstraints); - this.tpcUtils = new TPCUtils(this); + this.tpcUtils = new TPCUtils(this, { + audioQuality: options.audioQuality, + isP2P: this.isP2P, + videoQuality: options.videoQuality + }); this.updateLog = []; this.stats = {}; this.statsinterval = null; @@ -311,19 +312,12 @@ export default function TraceablePeerConnection( */ this._senderMaxHeights = new Map(); - /** - * Flag indicating bridge support for AV1 codec. On the bridge connection, it is supported only when support for - * Dependency Descriptor header extensions is offered by Jicofo. H.264 simulcast is also possible when these - * header extensions are negotiated. - */ - this._supportsDDHeaderExt = false; - /** * Holds the RTCRtpTransceiver mids that the local tracks are attached to, mapped per their * {@link JitsiLocalTrack.rtcId}. * @type {Map} */ - this._localTrackTransceiverMids = new Map(); + this.localTrackTransceiverMids = new Map(); /** * Holds the SSRC map for the local tracks mapped by their source names. @@ -505,20 +499,14 @@ TraceablePeerConnection.prototype.getConnectionState = function() { * Obtains the media direction for given {@link MediaType} that needs to be set on a p2p peerconnection's remote SDP * after a source-add or source-remove action. The method takes into account whether or not there are any * local tracks for the given media type. - * @param {MediaType} mediaType + * @param {MediaType} mediaType - The media type for which the direction is to be calculated. * @param {boolean} isAddOperation whether the direction is to be calculated after a source-add action. * @return {string} one of the SDP direction constants ('sendrecv, 'recvonly' etc.) which should be used when setting * local description on the peerconnection. * @private */ TraceablePeerConnection.prototype.getDesiredMediaDirection = function(mediaType, isAddOperation = false) { - const hasLocalSource = this.hasAnyTracksOfType(mediaType); - - if (isAddOperation) { - return hasLocalSource ? MediaDirection.SENDRECV : MediaDirection.SENDONLY; - } - - return hasLocalSource ? MediaDirection.RECVONLY : MediaDirection.INACTIVE; + return this.tpcUtils.getDesiredMediaDirection(mediaType, isAddOperation); }; /** @@ -675,7 +663,7 @@ TraceablePeerConnection.prototype.getAudioLevels = function(speakerList = []) { * @returns {boolean} */ TraceablePeerConnection.prototype.doesTrueSimulcast = function(localTrack) { - const currentCodec = this.getConfiguredVideoCodec(localTrack); + const currentCodec = this.tpcUtils.getConfiguredVideoCodec(localTrack); return this.isSpatialScalabilityOn() && this.tpcUtils.isRunningInSimulcastMode(currentCodec); }; @@ -723,21 +711,6 @@ TraceablePeerConnection.prototype.getLocalVideoTracks = function() { return this.getLocalTracks(MediaType.VIDEO); }; -/** - * Checks whether or not this {@link TraceablePeerConnection} instance contains any local tracks for given - * mediaType. - * - * @param {MediaType} mediaType - The media type. - * @return {boolean} - */ -TraceablePeerConnection.prototype.hasAnyTracksOfType = function(mediaType) { - if (!mediaType) { - throw new Error('"mediaType" is required'); - } - - return this.getLocalTracks(mediaType).length > 0; -}; - /** * Obtains all remote tracks currently known to this PeerConnection instance. * @@ -822,7 +795,7 @@ TraceablePeerConnection.prototype.getRemoteSourceInfoByParticipant = function(id * @returns {Object} */ TraceablePeerConnection.prototype.getTargetVideoBitrates = function(localTrack) { - const currentCodec = this.getConfiguredVideoCodec(localTrack); + const currentCodec = this.tpcUtils.getConfiguredVideoCodec(localTrack); return this.tpcUtils.codecSettings[currentCodec].maxBitratesVideo; }; @@ -1352,112 +1325,6 @@ TraceablePeerConnection.prototype._isSharingScreen = function() { return Boolean(tracks.find(track => track.videoType === VideoType.DESKTOP)); }; -/** - * Munges the order of the codecs in the SDP passed based on the preference - * set through config.js settings. All instances of the specified codec are - * moved up to the top of the list when it is preferred. The specified codec - * is deleted from the list if the configuration specifies that the codec be - * disabled. - * @param {RTCSessionDescription} description that needs to be munged. - * @returns {RTCSessionDescription} the munged description. - */ -TraceablePeerConnection.prototype._mungeCodecOrder = function(description) { - if (!this.codecSettings) { - return description; - } - - const parsedSdp = transform.parse(description.sdp); - const mLines = parsedSdp.media.filter(m => m.type === this.codecSettings.mediaType); - - if (!mLines.length) { - return description; - } - - for (const mLine of mLines) { - const currentCodecs = this.getConfiguredVideoCodecs(description); - - for (const codec of currentCodecs) { - if (this.isP2P) { - // 1. Strip the high profile H264 codecs on all clients. macOS started offering encoder for H.264 level - // 5.2 but a decoder only for level 3.1. Therfore, strip all main and high level codecs for H.264. - // 2. There are multiple VP9 payload types generated by the browser, more payload types are added if the - // endpoint doesn't have a local video source. Therefore, strip all the high profile codec variants - // for VP9 so that only one payload type for VP9 is negotiated between the peers. - if (codec === CodecMimeType.H264 || codec === CodecMimeType.VP9) { - SDPUtil.stripCodec(mLine, codec, true /* high profile */); - } - - // Do not negotiate ULPFEC and RED either. - if (codec === CodecMimeType.ULPFEC || codec === CodecMimeType.RED) { - SDPUtil.stripCodec(mLine, codec, false); - } - } - } - - // Reorder the codecs based on the preferred settings. - if (!this.usesCodecSelectionAPI()) { - for (const codec of this.codecSettings.codecList.slice().reverse()) { - SDPUtil.preferCodec(mLine, codec, this.isP2P); - } - } - } - - return new RTCSessionDescription({ - type: description.type, - sdp: transform.write(parsedSdp) - }); -}; - -/** - * Checks if the AV1 Dependency descriptors are negotiated on the bridge peerconnection and disables them when the - * codec selected is VP8 or VP9. - * - * @param {RTCSessionDescription} description that needs to be munged. - * @returns {RTCSessionDescription} the munged description. - */ -TraceablePeerConnection.prototype._updateAv1DdHeaders = function(description) { - const parsedSdp = transform.parse(description.sdp); - const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO); - - if (!mLines.length || !browser.supportsDDExtHeaders()) { - return description; - } - - mLines.forEach((mLine, idx) => { - const senderMids = Array.from(this._localTrackTransceiverMids.values()); - const isSender = senderMids.length - ? senderMids.find(mid => mLine.mid.toString() === mid.toString()) - : idx === 0; - const payload = mLine.payloads.split(' ')[0]; - let { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload)); - - codec = codec.toLowerCase(); - - if (isSender && mLine.ext?.length) { - const headerIndex = mLine.ext.findIndex(ext => ext.uri === DD_HEADER_EXT_URI); - const shouldNegotiateHeaderExts = codec === CodecMimeType.AV1 || codec === CodecMimeType.H264; - - if (!this._supportsDDHeaderExt && headerIndex >= 0) { - this._supportsDDHeaderExt = true; - } - - if (this._supportsDDHeaderExt && shouldNegotiateHeaderExts && headerIndex < 0) { - mLine.ext.push({ - value: DD_HEADER_EXT_ID, - uri: DD_HEADER_EXT_URI - }); - } else if (!shouldNegotiateHeaderExts && headerIndex >= 0) { - mLine.ext.splice(headerIndex, 1); - } - } - }); - - return new RTCSessionDescription({ - type: description.type, - sdp: transform.write(parsedSdp) - }); -}; - /** * Add {@link JitsiLocalTrack} to this TPC. * @param {JitsiLocalTrack} track @@ -1466,30 +1333,54 @@ TraceablePeerConnection.prototype._updateAv1DdHeaders = function(description) { */ TraceablePeerConnection.prototype.addTrack = function(track, isInitiator = false) { const rtcId = track.rtcId; - let transceiver; - logger.info(`${this} adding ${track}`); if (this.localTracks.has(rtcId)) { - return Promise.reject(new Error(`${track} is already in ${this}`)); } - this.localTracks.set(rtcId, track); + logger.info(`${this} adding ${track}`); const webrtcStream = track.getOriginalStream(); + const mediaStreamTrack = track.getTrack(); + let transceiver; + + if (isInitiator) { + const streams = []; - try { - transceiver = this.tpcUtils.addTrack(track, isInitiator); - } catch (error) { - logger.error(`${this} Adding track=${track} failed: ${error?.message}`); + webrtcStream && streams.push(webrtcStream); - return Promise.reject(error); + // Use pc.addTransceiver() for the initiator case when local tracks are getting added + // to the peerconnection before a session-initiate is sent over to the peer. + const transceiverInit = { + direction: MediaDirection.SENDRECV, + streams, + sendEncodings: [] + }; + + if (!browser.isFirefox()) { + transceiverInit.sendEncodings = this.tpcUtils.getStreamEncodings(track); + } + + try { + transceiver = this.peerconnection.addTransceiver(mediaStreamTrack, transceiverInit); + } catch (error) { + return Promise.reject(new Error(`${this} failed to create transceiver for ${track} - ${error}`)); + } + } else { + // Use pc.addTrack() for responder case so that we can re-use the m-lines that were created + // when setRemoteDescription was called. pc.addTrack() automatically attaches to any existing + // unused "recv-only" transceiver. + const sender = this.peerconnection.addTrack(mediaStreamTrack); + + // Find the corresponding transceiver that the track was attached to. + transceiver = this.peerconnection.getTransceivers().find(t => t.sender === sender); } if (transceiver?.mid) { - this._localTrackTransceiverMids.set(track.rtcId, transceiver.mid.toString()); + this.localTrackTransceiverMids.set(track.rtcId, transceiver.mid.toString()); } if (track) { + this.localTracks.set(rtcId, track); if (track.isAudioTrack()) { this._hasHadAudioTrack = true; } else { @@ -1501,7 +1392,7 @@ TraceablePeerConnection.prototype.addTrack = function(track, isInitiator = false // On Firefox, the encodings have to be configured on the sender only after the transceiver is created. if (browser.isFirefox()) { - promiseChain = promiseChain.then(() => webrtcStream && this.tpcUtils.setEncodings(track)); + promiseChain = promiseChain.then(() => webrtcStream && this._setEncodings(track)); } return promiseChain; @@ -1531,7 +1422,7 @@ TraceablePeerConnection.prototype.addTrackToPc = function(track) { return Promise.reject('Stream not found'); } - return this.tpcUtils.replaceTrack(null, track).then(() => { + return this.replaceTrack(null, track, true /* isMuteOperation */).then(() => { if (track) { if (track.isAudioTrack()) { this._hasHadAudioTrack = true; @@ -1567,42 +1458,6 @@ TraceablePeerConnection.prototype._assertTrackBelongs = function( return doesBelong; }; -/** - * Returns the codec that is configured on the client as the preferred video codec for the given local video track. - * - * @param {JitsiLocalTrack} localTrack - The local video track. - * @returns {CodecMimeType} The codec that is set as the preferred codec for the given local video track. - * - */ -TraceablePeerConnection.prototype.getConfiguredVideoCodec = function(localTrack) { - const localVideoTrack = localTrack ?? this.getLocalVideoTracks()[0]; - const rtpSender = this.findSenderForTrack(localVideoTrack.getTrack()); - - if (this.usesCodecSelectionAPI() && rtpSender) { - const { codecs } = rtpSender.getParameters(); - - return codecs[0].mimeType.split('/')[1].toLowerCase(); - } - - const sdp = this.remoteDescription?.sdp; - const defaultCodec = CodecMimeType.VP8; - - if (!sdp) { - return defaultCodec; - } - const parsedSdp = transform.parse(sdp); - const mLine = parsedSdp.media - .find(m => m.mid.toString() === this._localTrackTransceiverMids.get(localVideoTrack.rtcId)); - const payload = mLine.payloads.split(' ')[0]; - const { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload)); - - if (codec) { - return Object.values(CodecMimeType).find(value => value === codec.toLowerCase()); - } - - return defaultCodec; -}; - /** * Returns the codecs in the current order of preference as configured on the peerconnection. * @@ -1610,18 +1465,7 @@ TraceablePeerConnection.prototype.getConfiguredVideoCodec = function(localTrack) * @returns {Array} */ TraceablePeerConnection.prototype.getConfiguredVideoCodecs = function(description) { - const currentSdp = description?.sdp ?? this.localDescription?.sdp; - - if (!currentSdp) { - return []; - } - const parsedSdp = transform.parse(currentSdp); - const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO); - const codecs = new Set(mLine.rtp - .filter(pt => pt.codec.toLowerCase() !== 'rtx') - .map(pt => pt.codec.toLowerCase())); - - return Array.from(codecs); + return this.tpcUtils.getConfiguredVideoCodecs(description); }; /** @@ -1725,8 +1569,8 @@ TraceablePeerConnection.prototype.processLocalSdpForTransceiverInfo = function(l const mLines = parsedSdp.media.filter(mline => mline.type === mediaType); tracks.forEach((track, idx) => { - if (!this._localTrackTransceiverMids.has(track.rtcId)) { - this._localTrackTransceiverMids.set(track.rtcId, mLines[idx].mid.toString()); + if (!this.localTrackTransceiverMids.has(track.rtcId)) { + this.localTrackTransceiverMids.set(track.rtcId, mLines[idx].mid.toString()); } }); }); @@ -1740,11 +1584,11 @@ TraceablePeerConnection.prototype.processLocalSdpForTransceiverInfo = function(l * * @param {JitsiLocalTrack|null} oldTrack - The current track in use to be replaced on the pc. * @param {JitsiLocalTrack|null} newTrack - The new track to be used. - * + * @param {boolean} isMuteOperation - Whether the operation is a mute/unmute operation. * @returns {Promise} - If the promise resolves with true, renegotiation will be needed. * Otherwise no renegotiation is needed. */ -TraceablePeerConnection.prototype.replaceTrack = function(oldTrack, newTrack) { +TraceablePeerConnection.prototype.replaceTrack = function(oldTrack, newTrack, isMuteOperation = false) { if (!(oldTrack || newTrack)) { logger.info(`${this} replaceTrack called with no new track and no old track`); @@ -1753,11 +1597,68 @@ TraceablePeerConnection.prototype.replaceTrack = function(oldTrack, newTrack) { logger.info(`${this} TPC.replaceTrack old=${oldTrack}, new=${newTrack}`); - return this.tpcUtils.replaceTrack(oldTrack, newTrack) - .then(transceiver => { + let transceiver; + const mediaType = newTrack?.getType() ?? oldTrack?.getType(); + const localTracks = this.getLocalTracks(mediaType); + const track = newTrack?.getTrack() ?? null; + const isNewLocalSource = localTracks?.length + && !oldTrack + && newTrack + && !localTracks.find(t => t === newTrack); + + // If old track exists, replace the track on the corresponding sender. + if (oldTrack && !oldTrack.isMuted()) { + transceiver = this.peerconnection.getTransceivers().find(t => t.sender.track === oldTrack.getTrack()); + + // Find the first recvonly transceiver when more than one track of the same media type is being added to the pc. + // As part of the track addition, a new m-line was added to the remote description with direction set to + // recvonly. + } else if (isNewLocalSource) { + transceiver = this.peerconnection.getTransceivers().find( + t => t.receiver.track.kind === mediaType + && t.direction === MediaDirection.RECVONLY + + // Re-use any existing recvonly transceiver (if available) for p2p case. + && ((this.isP2P && t.currentDirection === MediaDirection.RECVONLY) + || (t.currentDirection === MediaDirection.INACTIVE && !t.stopped))); + + // For mute/unmute operations, find the transceiver based on the track index in the source name if present, + // otherwise it is assumed to be the first local track that was added to the peerconnection. + } else { + transceiver = this.peerconnection.getTransceivers().find(t => t.receiver.track.kind === mediaType); + const sourceName = newTrack?.getSourceName() ?? oldTrack?.getSourceName(); + + if (sourceName) { + const trackIndex = getSourceIndexFromSourceName(sourceName); + + if (this.isP2P) { + transceiver = this.peerconnection.getTransceivers() + .filter(t => t.receiver.track.kind === mediaType)[trackIndex]; + } else if (oldTrack) { + const transceiverMid = this.localTrackTransceiverMids.get(oldTrack.rtcId); + + transceiver = this.peerconnection.getTransceivers().find(t => t.mid === transceiverMid); + } else if (trackIndex) { + transceiver = this.peerconnection.getTransceivers() + .filter(t => t.receiver.track.kind === mediaType + && t.direction !== MediaDirection.RECVONLY)[trackIndex]; + } + } + } + + if (!transceiver) { + return Promise.reject( + new Error(`Replace track failed - no transceiver for old: ${oldTrack}, new: ${newTrack}`)); + } + + return transceiver.sender.replaceTrack(track) + .then(() => { + if (isMuteOperation) { + return Promise.resolve(); + } if (oldTrack) { this.localTracks.delete(oldTrack.rtcId); - this._localTrackTransceiverMids.delete(oldTrack.rtcId); + this.localTrackTransceiverMids.delete(oldTrack.rtcId); } if (newTrack) { @@ -1766,7 +1667,7 @@ TraceablePeerConnection.prototype.replaceTrack = function(oldTrack, newTrack) { } else { this._hasHadVideoTrack = true; } - this._localTrackTransceiverMids.set(newTrack.rtcId, transceiver?.mid?.toString()); + this.localTrackTransceiverMids.set(newTrack.rtcId, transceiver?.mid?.toString()); this.localTracks.set(newTrack.rtcId, newTrack); } @@ -1804,7 +1705,7 @@ TraceablePeerConnection.prototype.replaceTrack = function(oldTrack, newTrack) { const configureEncodingsPromise = !newTrack || browser.usesSdpMungingForSimulcast() ? Promise.resolve() - : this.tpcUtils.setEncodings(newTrack); + : this._setEncodings(newTrack); return configureEncodingsPromise.then(() => this.isP2P); }); @@ -1827,7 +1728,7 @@ TraceablePeerConnection.prototype.removeTrackFromPc = function(localTrack) { return Promise.reject('Track not found in the peerconnection'); } - return this.tpcUtils.replaceTrack(localTrack, null).then(() => false); + return this.replaceTrack(localTrack, null, true /* isMuteOperation */).then(() => false); }; /** @@ -1937,81 +1838,6 @@ TraceablePeerConnection.prototype._getPreferredCodecForScreenshare = function(de return defaultCodec; }; -/** - * Munges the stereo flag as well as the opusMaxAverageBitrate in the SDP, based - * on values set through config.js, if present. - * - * @param {RTCSessionDescription} description that needs to be munged. - * @returns {RTCSessionDescription} the munged description. - */ -TraceablePeerConnection.prototype._mungeOpus = function(description) { - const { audioQuality } = this.options; - - if (!audioQuality?.enableOpusDtx && !audioQuality?.stereo && !audioQuality?.opusMaxAverageBitrate) { - return description; - } - - const parsedSdp = transform.parse(description.sdp); - const mLines = parsedSdp.media; - - for (const mLine of mLines) { - if (mLine.type === 'audio') { - const { payload } = mLine.rtp.find(protocol => protocol.codec === CodecMimeType.OPUS); - - if (!payload) { - // eslint-disable-next-line no-continue - continue; - } - - let fmtpOpus = mLine.fmtp.find(protocol => protocol.payload === payload); - - if (!fmtpOpus) { - fmtpOpus = { - payload, - config: '' - }; - } - - const fmtpConfig = transform.parseParams(fmtpOpus.config); - let sdpChanged = false; - - if (audioQuality?.stereo) { - fmtpConfig.stereo = 1; - sdpChanged = true; - } - - if (audioQuality?.opusMaxAverageBitrate) { - fmtpConfig.maxaveragebitrate = audioQuality.opusMaxAverageBitrate; - sdpChanged = true; - } - - // On Firefox, the OpusDtx enablement has no effect - if (!browser.isFirefox() && audioQuality?.enableOpusDtx) { - fmtpConfig.usedtx = 1; - sdpChanged = true; - } - - if (!sdpChanged) { - // eslint-disable-next-line no-continue - continue; - } - - let mungedConfig = ''; - - for (const key of Object.keys(fmtpConfig)) { - mungedConfig += `${key}=${fmtpConfig[key]}; `; - } - - fmtpOpus.config = mungedConfig.trim(); - } - } - - return new RTCSessionDescription({ - type: description.type, - sdp: transform.write(parsedSdp) - }); -}; - /** * Sets up the _dtlsTransport object and initializes callbacks for it. */ @@ -2036,70 +1862,6 @@ TraceablePeerConnection.prototype._initializeDtlsTransport = function() { } }; -/** - * Sets the max bitrates on the video m-lines when VP9/AV1 is the selected codec. - * - * @param {RTCSessionDescription} description - The local description that needs to be munged. - * @param {boolean} isLocalSdp - Whether the max bitrate (via b=AS line in SDP) is set on local SDP. - * @returns RTCSessionDescription - */ -TraceablePeerConnection.prototype._setMaxBitrates = function(description, isLocalSdp = false) { - if (!this.codecSettings) { - return description; - } - const parsedSdp = transform.parse(description.sdp); - - // Find all the m-lines associated with the local sources. - const direction = isLocalSdp ? MediaDirection.RECVONLY : MediaDirection.SENDONLY; - const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO && m.direction !== direction); - const currentCodec = this.codecSettings.codecList[0]; - const codecScalabilityModeSettings = this.tpcUtils.codecSettings[currentCodec]; - - for (const mLine of mLines) { - const isDoingVp9KSvc = currentCodec === CodecMimeType.VP9 - && !codecScalabilityModeSettings.scalabilityModeEnabled; - const localTrack = this.getLocalVideoTracks() - .find(track => this._localTrackTransceiverMids.get(track.rtcId) === mLine.mid.toString()); - - if (localTrack - && (isDoingVp9KSvc - - // Setting bitrates in the SDP for SVC codecs is no longer needed in the newer versions where - // maxBitrates from the RTCRtpEncodingParameters directly affect the target bitrate for the encoder. - || (this.tpcUtils._isRunningInFullSvcMode(currentCodec) && !this.usesCodecSelectionAPI()))) { - let maxBitrate; - - if (localTrack.getVideoType() === VideoType.DESKTOP) { - maxBitrate = codecScalabilityModeSettings.maxBitratesVideo.ssHigh; - } else { - const { level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= localTrack.getCaptureResolution()); - - maxBitrate = codecScalabilityModeSettings.maxBitratesVideo[level]; - } - - const limit = Math.floor(maxBitrate / 1000); - - // Use only the highest spatial layer bitrates for now as there is no API available yet for configuring - // the bitrates on the individual SVC layers. - mLine.bandwidth = [ { - type: 'AS', - limit - } ]; - } else { - // Clear the bandwidth limit in SDP when VP9 is no longer the preferred codec. - // This is needed on react native clients as react-native-webrtc returns the - // SDP that the application passed instead of returning the SDP off the native side. - // This line automatically gets cleared on web on every renegotiation. - mLine.bandwidth = undefined; - } - } - - return new RTCSessionDescription({ - type: description.type, - sdp: transform.write(parsedSdp) - }); -}; - /** * Returns the expected send resolution for a local video track based on what encodings are currently active. * @@ -2144,17 +1906,61 @@ TraceablePeerConnection.prototype.calculateExpectedSendResolution = function(loc */ TraceablePeerConnection.prototype.configureAudioSenderEncodings = function(localAudioTrack = null) { if (localAudioTrack) { - return this.tpcUtils.setEncodings(localAudioTrack); + return this._setEncodings(localAudioTrack); } const promises = []; for (const track of this.getLocalTracks(MediaType.AUDIO)) { - promises.push(this.tpcUtils.setEncodings(track)); + promises.push(this._setEncodings(track)); } return Promise.allSettled(promises); }; +/** + * Configures the RTCRtpEncodingParameters of the outbound rtp stream associated with the given track. + * + * @param {JitsiLocalTracj} localTrack - The local track whose outbound stream needs to be configured. + * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise. + */ +TraceablePeerConnection.prototype._configureSenderEncodings = function(localTrack) { + const mediaType = localTrack.getType(); + const transceiver = localTrack?.track && localTrack.getOriginalStream() + ? this.peerconnection.getTransceivers().find(t => t.sender?.track?.id === localTrack.getTrackId()) + : this.peerconnection.getTransceivers().find(t => t.receiver?.track?.kind === mediaType); + const parameters = transceiver?.sender?.getParameters(); + + // Resolve if the encodings are not available yet. This happens immediately after the track is added to the + // peerconnection on chrome in unified-plan. It is ok to ignore and not report the error here since the + // action that triggers 'addTrack' (like unmute) will also configure the encodings and set bitrates after that. + if (!parameters?.encodings?.length) { + return Promise.resolve(); + } + + parameters.encodings = this.tpcUtils.getStreamEncodings(localTrack); + + return transceiver.sender.setParameters(parameters); +}; + +/** + * Enables/disables the streams by changing the active field on RTCRtpEncodingParameters for a given RTCRtpSender. + * + * @param {RTCRtpSender} sender - the sender associated with a MediaStreamTrack. + * @param {boolean} enable - whether the streams needs to be enabled or disabled. + * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise. + */ +TraceablePeerConnection.prototype._enableSenderEncodings = function(sender, enable) { + const parameters = sender.getParameters(); + + if (parameters?.encodings?.length) { + for (const encoding of parameters.encodings) { + encoding.active = enable; + } + } + + return sender.setParameters(parameters); +}; + /** * Configures the stream encodings depending on the video type, scalability mode and the bitrate settings for the codec * that is currently selected. @@ -2182,20 +1988,50 @@ TraceablePeerConnection.prototype.configureVideoSenderEncodings = function(local return Promise.allSettled(promises); }; -TraceablePeerConnection.prototype.setLocalDescription = function(description) { - let localDescription = description; +/** + * Set the simulcast stream encoding properties on the RTCRtpSender. + * + * @param {JitsiLocalTrack} localTrack - the current track in use for which the encodings are to be set. + * @returns {Promise} - resolved when done. + */ +TraceablePeerConnection.prototype._setEncodings = function(localTrack) { + if (localTrack.getType() === MediaType.VIDEO) { + return this._updateVideoSenderParameters(() => this._configureSenderEncodings(localTrack)); + } - this.trace('setLocalDescription::preTransform', dumpSDP(localDescription)); + return this._configureSenderEncodings(localTrack); +}; - // Munge stereo flag and opusMaxAverageBitrate based on config.js - localDescription = this._mungeOpus(localDescription); +/** + * Munges the provided description to update the codec order, set the max bitrates (for VP9 K-SVC), set stereo flag + * and update the DD Header extensions for AV1. + * + * @param {RTCSessionDescription} description - The description to be munged. + * @returns {RTCSessionDescription} - The munged description. + */ +TraceablePeerConnection.prototype._mungeDescription = function(description) { + let mungedDescription = description; + + this.trace('RTCSessionDescription::preTransform', dumpSDP(description)); + mungedDescription = this.tpcUtils.mungeOpus(description); + mungedDescription = this.tpcUtils.mungeCodecOrder(mungedDescription); + mungedDescription = this.tpcUtils.setMaxBitrates(mungedDescription, true); + mungedDescription = this.tpcUtils.updateAv1DdHeaders(mungedDescription); + this.trace('RTCSessionDescription::postTransform', dumpSDP(mungedDescription)); + + return mungedDescription; +}; - // Munge the order of the codecs based on the preferences set through config.js. - localDescription = this._mungeCodecOrder(localDescription); - localDescription = this._setMaxBitrates(localDescription, true); - localDescription = this._updateAv1DdHeaders(localDescription); +/** + * Sets the local description on the peerconnection. + * + * @param {RTCSessionDescription} description - The local description to be set. + * @returns {Promise} - Resolved when the operation is successful and rejected with an error otherwise. + */ +TraceablePeerConnection.prototype.setLocalDescription = function(description) { + let localDescription = description; - this.trace('setLocalDescription::postTransform', dumpSDP(localDescription)); + localDescription = this._mungeDescription(localDescription); return new Promise((resolve, reject) => { this.peerconnection.setLocalDescription(localDescription) @@ -2219,14 +2055,15 @@ TraceablePeerConnection.prototype.setLocalDescription = function(description) { }); }; +/** + * Sets the remote description on the peerconnection. + * + * @param {RTCSessionDescription} description - The remote description to be set. + * @returns {Promise} - Resolved when the operation is successful and rejected with an error otherwise. + */ TraceablePeerConnection.prototype.setRemoteDescription = function(description) { let remoteDescription = description; - this.trace('setRemoteDescription::preTransform', dumpSDP(description)); - - // Munge stereo flag and opusMaxAverageBitrate based on config.js - remoteDescription = this._mungeOpus(remoteDescription); - if (this.isSpatialScalabilityOn()) { remoteDescription = this.tpcUtils.insertUnifiedPlanSimulcastReceive(remoteDescription); this.trace('setRemoteDescription::postTransform (sim receive)', dumpSDP(remoteDescription)); @@ -2234,11 +2071,7 @@ TraceablePeerConnection.prototype.setRemoteDescription = function(description) { remoteDescription = this.tpcUtils.ensureCorrectOrderOfSsrcs(remoteDescription); this.trace('setRemoteDescription::postTransform (correct ssrc order)', dumpSDP(remoteDescription)); - // Munge the order of the codecs based on the preferences set through config.js. - remoteDescription = this._mungeCodecOrder(remoteDescription); - remoteDescription = this._setMaxBitrates(remoteDescription); - remoteDescription = this._updateAv1DdHeaders(remoteDescription); - this.trace('setRemoteDescription::postTransform (munge codec order)', dumpSDP(remoteDescription)); + remoteDescription = this._mungeDescription(remoteDescription); return new Promise((resolve, reject) => { this.peerconnection.setRemoteDescription(remoteDescription) @@ -2343,7 +2176,7 @@ TraceablePeerConnection.prototype._updateVideoSenderEncodings = function(frameHe parameters.degradationPreference = preference; // Calculate the encodings active state based on the resolution requested by the bridge. - const codecForCamera = preferredCodec ?? this.getConfiguredVideoCodec(localVideoTrack); + const codecForCamera = preferredCodec ?? this.tpcUtils.getConfiguredVideoCodec(localVideoTrack); const codec = isScreensharingTrack ? this._getPreferredCodecForScreenshare(codecForCamera) : codecForCamera; const activeState = this.tpcUtils.calculateEncodingsActiveState(localVideoTrack, codec, frameHeight); let bitrates = this.tpcUtils.calculateEncodingsBitrates(localVideoTrack, codec, frameHeight); @@ -2435,6 +2268,46 @@ TraceablePeerConnection.prototype._updateVideoSenderEncodings = function(frameHe }); }; +/** + * Resumes or suspends media on the peerconnection by setting the active state on RTCRtpEncodingParameters + * associated with all the senders that have a track attached to it. + * + * @param {boolean} enable - whether outgoing media needs to be enabled or disabled. + * @param {string} mediaType - media type, 'audio' or 'video', if neither is passed, all outgoing media will either + * be enabled or disabled. + * @returns {Promise} - A promise that is resolved when the change is succesful on all the senders, rejected + * otherwise. + */ +TraceablePeerConnection.prototype.setMediaTransferActive = function(enable, mediaType) { + logger.info(`${this} ${enable ? 'Resuming' : 'Suspending'} media transfer.`); + + const senders = this.peerconnection.getSenders() + .filter(s => Boolean(s.track) && (!mediaType || s.track.kind === mediaType)); + const promises = []; + + for (const sender of senders) { + if (sender.track.kind === MediaType.VIDEO) { + promises.push(this._updateVideoSenderParameters(() => this._enableSenderEncodings(sender, enable))); + } else { + promises.push(this._enableSenderEncodings(sender, enable)); + } + } + + return Promise.allSettled(promises) + .then(settledResult => { + const errors = settledResult + .filter(result => result.status === 'rejected') + .map(result => result.reason); + + if (errors.length) { + return Promise.reject(new Error('Failed to change encodings on the RTCRtpSenders' + + `${errors.join(' ')}`)); + } + + return Promise.resolve(); + }); +}; + /** * Enables/disables outgoing video media transmission on this peer connection. When disabled the stream encoding's * active state is enabled or disabled to send or stop the media. @@ -2450,7 +2323,7 @@ TraceablePeerConnection.prototype.setVideoTransferActive = function(active) { this.videoTransferActive = active; if (changed) { - return this.tpcUtils.setMediaTransferActive(active, MediaType.VIDEO); + return this.setMediaTransferActive(active, MediaType.VIDEO); } return Promise.resolve(); diff --git a/modules/xmpp/JingleSessionPC.js b/modules/xmpp/JingleSessionPC.js index 9b5044275a..e38b285a12 100644 --- a/modules/xmpp/JingleSessionPC.js +++ b/modules/xmpp/JingleSessionPC.js @@ -2103,7 +2103,7 @@ export default class JingleSessionPC extends JingleSession { return Promise.resolve(); } - return this.peerconnection.tpcUtils.setMediaTransferActive(active) + return this.peerconnection.setMediaTransferActive(active) .then(() => { this.peerconnection.audioTransferActive = active; this.peerconnection.videoTransferActive = active; diff --git a/types/hand-crafted/modules/RTC/TPCUtils.d.ts b/types/hand-crafted/modules/RTC/TPCUtils.d.ts index 522f1b8889..f2fd1be89f 100644 --- a/types/hand-crafted/modules/RTC/TPCUtils.d.ts +++ b/types/hand-crafted/modules/RTC/TPCUtils.d.ts @@ -9,7 +9,6 @@ export default class TPCUtils { getLocalStreamHeightConstraints: ( localTrack: JitsiLocalTrack ) => number[]; removeTrackMute: ( localTrack: JitsiLocalTrack ) => Promise; replaceTrack: ( oldTrack: JitsiLocalTrack, newTrack: JitsiLocalTrack ) => Promise; - setEncodings: ( track: JitsiLocalTrack ) => Promise; setMediaTransferActive: ( active: boolean ) => void; setVideoTransferActive: ( active: boolean ) => void; updateEncodingsResolution: ( parameters: RTCRtpEncodingParameters ) => void; From 0bd87d4efb2a1ba7756b05e11ae4e523cb08f702 Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Thu, 21 Nov 2024 11:05:42 -0500 Subject: [PATCH 2/6] ref(TPC): Convert to async functions. --- modules/RTC/TraceablePeerConnection.js | 29 +++++++++----------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/modules/RTC/TraceablePeerConnection.js b/modules/RTC/TraceablePeerConnection.js index 12ee794642..aed187393c 100644 --- a/modules/RTC/TraceablePeerConnection.js +++ b/modules/RTC/TraceablePeerConnection.js @@ -1331,11 +1331,11 @@ TraceablePeerConnection.prototype._isSharingScreen = function() { * @param {boolean} isInitiator indicates if the endpoint is the offerer. * @returns {Promise} - resolved when done. */ -TraceablePeerConnection.prototype.addTrack = function(track, isInitiator = false) { +TraceablePeerConnection.prototype.addTrack = async function(track, isInitiator = false) { const rtcId = track.rtcId; if (this.localTracks.has(rtcId)) { - return Promise.reject(new Error(`${track} is already in ${this}`)); + throw new Error(`${track} is already in ${this}`); } logger.info(`${this} adding ${track}`); @@ -1360,11 +1360,7 @@ TraceablePeerConnection.prototype.addTrack = function(track, isInitiator = false transceiverInit.sendEncodings = this.tpcUtils.getStreamEncodings(track); } - try { - transceiver = this.peerconnection.addTransceiver(mediaStreamTrack, transceiverInit); - } catch (error) { - return Promise.reject(new Error(`${this} failed to create transceiver for ${track} - ${error}`)); - } + transceiver = this.peerconnection.addTransceiver(mediaStreamTrack, transceiverInit); } else { // Use pc.addTrack() for responder case so that we can re-use the m-lines that were created // when setRemoteDescription was called. pc.addTrack() automatically attaches to any existing @@ -1388,14 +1384,10 @@ TraceablePeerConnection.prototype.addTrack = function(track, isInitiator = false } } - let promiseChain = Promise.resolve(); - // On Firefox, the encodings have to be configured on the sender only after the transceiver is created. - if (browser.isFirefox()) { - promiseChain = promiseChain.then(() => webrtcStream && this._setEncodings(track)); + if (browser.isFirefox() && webrtcStream) { + await this._setEncodings(track); } - - return promiseChain; }; /** @@ -1923,7 +1915,7 @@ TraceablePeerConnection.prototype.configureAudioSenderEncodings = function(local * @param {JitsiLocalTracj} localTrack - The local track whose outbound stream needs to be configured. * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise. */ -TraceablePeerConnection.prototype._configureSenderEncodings = function(localTrack) { +TraceablePeerConnection.prototype._configureSenderEncodings = async function(localTrack) { const mediaType = localTrack.getType(); const transceiver = localTrack?.track && localTrack.getOriginalStream() ? this.peerconnection.getTransceivers().find(t => t.sender?.track?.id === localTrack.getTrackId()) @@ -1934,12 +1926,11 @@ TraceablePeerConnection.prototype._configureSenderEncodings = function(localTrac // peerconnection on chrome in unified-plan. It is ok to ignore and not report the error here since the // action that triggers 'addTrack' (like unmute) will also configure the encodings and set bitrates after that. if (!parameters?.encodings?.length) { - return Promise.resolve(); + return; } parameters.encodings = this.tpcUtils.getStreamEncodings(localTrack); - - return transceiver.sender.setParameters(parameters); + await transceiver.sender.setParameters(parameters); }; /** @@ -1949,7 +1940,7 @@ TraceablePeerConnection.prototype._configureSenderEncodings = function(localTrac * @param {boolean} enable - whether the streams needs to be enabled or disabled. * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise. */ -TraceablePeerConnection.prototype._enableSenderEncodings = function(sender, enable) { +TraceablePeerConnection.prototype._enableSenderEncodings = async function(sender, enable) { const parameters = sender.getParameters(); if (parameters?.encodings?.length) { @@ -1958,7 +1949,7 @@ TraceablePeerConnection.prototype._enableSenderEncodings = function(sender, enab } } - return sender.setParameters(parameters); + await sender.setParameters(parameters); }; /** From de4e1a5b5ebc7c98fd7c0e6978b2f997df1b94fc Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Thu, 21 Nov 2024 11:52:36 -0500 Subject: [PATCH 3/6] ref(TPC) Pass bare objects to methods that take SDP instead of passing an instance of RTCSessionDescription. The constructor for RTCSessionDescription has been deprecated - https://developer.mozilla.org/en-US/docs/Web/API/RTCSessionDescription/RTCSessionDescription. --- modules/RTC/TPCUtils.js | 24 ++++++++++++------------ modules/RTC/TraceablePeerConnection.js | 12 ++++++------ modules/sdp/LocalSdpMunger.js | 4 ++-- modules/sdp/SdpSimulcast.ts | 4 ++-- modules/xmpp/JingleSessionPC.js | 8 ++++---- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/modules/RTC/TPCUtils.js b/modules/RTC/TPCUtils.js index e6922809c5..b6e07d1d41 100644 --- a/modules/RTC/TPCUtils.js +++ b/modules/RTC/TPCUtils.js @@ -470,10 +470,10 @@ export class TPCUtils { mLine.ssrcs = reorderedSsrcs; }); - return new RTCSessionDescription({ + return { type: description.type, sdp: transform.write(parsedSdp) - }); + }; } /** @@ -626,10 +626,10 @@ export class TPCUtils { } }); - return new RTCSessionDescription({ + return { type: desc.type, sdp: transform.write(sdp) - }); + }; } /** @@ -703,10 +703,10 @@ export class TPCUtils { } } - return new RTCSessionDescription({ + return { type: description.type, sdp: transform.write(parsedSdp) - }); + }; } /** @@ -776,10 +776,10 @@ export class TPCUtils { fmtpOpus.config = mungedConfig.trim(); } - return new RTCSessionDescription({ + return { type: description.type, sdp: transform.write(parsedSdp) - }); + }; } /** @@ -842,10 +842,10 @@ export class TPCUtils { } } - return new RTCSessionDescription({ + return { type: description.type, sdp: transform.write(parsedSdp) - }); + }; } /** @@ -892,9 +892,9 @@ export class TPCUtils { } }); - return new RTCSessionDescription({ + return { type: description.type, sdp: transform.write(parsedSdp) - }); + }; } } diff --git a/modules/RTC/TraceablePeerConnection.js b/modules/RTC/TraceablePeerConnection.js index aed187393c..4f95e91c08 100644 --- a/modules/RTC/TraceablePeerConnection.js +++ b/modules/RTC/TraceablePeerConnection.js @@ -1233,10 +1233,10 @@ TraceablePeerConnection.prototype._injectSsrcGroupForUnifiedSimulcast = function } } - return new RTCSessionDescription({ + return { type: desc.type, sdp: transform.write(sdp) - }); + }; }; /* eslint-disable-next-line vars-on-top */ @@ -1797,10 +1797,10 @@ TraceablePeerConnection.prototype._adjustRemoteMediaDirection = function(remoteD }); }); - return new RTCSessionDescription({ + return { type: remoteDescription.type, sdp: transformer.toRawSDP() - }); + }; }; /** @@ -2458,10 +2458,10 @@ TraceablePeerConnection.prototype._createOfferOrAnswer = function(isOffer, const if (!this.options.disableRtx && browser.usesSdpMungingForSimulcast()) { // eslint-disable-next-line no-param-reassign - resultSdp = new RTCSessionDescription({ + resultSdp = { type: resultSdp.type, sdp: this.rtxModifier.modifyRtxSsrcs(resultSdp.sdp) - }); + }; this.trace( `create${logName}` diff --git a/modules/sdp/LocalSdpMunger.js b/modules/sdp/LocalSdpMunger.js index 3da390b7c5..2f688ce185 100644 --- a/modules/sdp/LocalSdpMunger.js +++ b/modules/sdp/LocalSdpMunger.js @@ -144,9 +144,9 @@ export default class LocalSdpMunger { this._transformMediaIdentifiers(videoMLine, ssrcMap); } - return new RTCSessionDescription({ + return { type: sessionDesc.type, sdp: transformer.toRawSDP() - }); + }; } } diff --git a/modules/sdp/SdpSimulcast.ts b/modules/sdp/SdpSimulcast.ts index 2811ac3c41..65204fc10e 100644 --- a/modules/sdp/SdpSimulcast.ts +++ b/modules/sdp/SdpSimulcast.ts @@ -227,9 +227,9 @@ export default class SdpSimulcast { } } - return new RTCSessionDescription({ + return { type: description.type, sdp: transform.write(session) - }); + }; } } diff --git a/modules/xmpp/JingleSessionPC.js b/modules/xmpp/JingleSessionPC.js index e38b285a12..2d836c34b7 100644 --- a/modules/xmpp/JingleSessionPC.js +++ b/modules/xmpp/JingleSessionPC.js @@ -655,10 +655,10 @@ export default class JingleSessionPC extends JingleSession { return Promise.reject(error); } - const remoteDescription = new RTCSessionDescription({ + const remoteDescription = { type: 'offer', sdp: remoteSdp - }); + }; const oldLocalSDP = this.peerconnection.localDescription.sdp; @@ -2062,10 +2062,10 @@ export default class JingleSessionPC extends JingleSession { const newRemoteSdp = this._processNewJingleOfferIq(jingleAnswer); const oldLocalSdp = new SDP(this.peerconnection.localDescription.sdp); - const remoteDescription = new RTCSessionDescription({ + const remoteDescription = { type: 'answer', sdp: newRemoteSdp.raw - }); + }; this.peerconnection.setRemoteDescription(remoteDescription) .then(() => { From e6670103b2e503f24ab5ff27a2eda749f1b44985 Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Thu, 21 Nov 2024 11:58:39 -0500 Subject: [PATCH 4/6] ref(TPCUtils) Make options default to {}. --- modules/RTC/TPCUtils.js | 6 +++++- modules/RTC/TPCUtils.spec.js | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/RTC/TPCUtils.js b/modules/RTC/TPCUtils.js index b6e07d1d41..68f4febd25 100644 --- a/modules/RTC/TPCUtils.js +++ b/modules/RTC/TPCUtils.js @@ -31,8 +31,12 @@ export class TPCUtils { * Creates a new instance for a given TraceablePeerConnection * * @param peerconnection - the tpc instance for which we have utility functions. + * @param options - additional options that can be passed to the utility functions. + * @param options.audioQuality - the audio quality settings that are used to calculate the audio codec parameters. + * @param options.isP2P - whether the connection is a P2P connection. + * @param options.videoQuality - the video quality settings that are used to calculate the encoding parameters. */ - constructor(peerconnection, options) { + constructor(peerconnection, options = {}) { this.pc = peerconnection; this.options = options; this.codecSettings = cloneDeep(STANDARD_CODEC_SETTINGS); diff --git a/modules/RTC/TPCUtils.spec.js b/modules/RTC/TPCUtils.spec.js index 9faf4d5b6c..0231557917 100644 --- a/modules/RTC/TPCUtils.spec.js +++ b/modules/RTC/TPCUtils.spec.js @@ -99,7 +99,7 @@ describe('TPCUtils', () => { it('sort ssrcs in case the first ssrc in the SIM group is not present at the top', () => { const pc = new MockPeerConnection(); - const tpcUtils = new TPCUtils(pc, { }); + const tpcUtils = new TPCUtils(pc); const source = new RTCSessionDescription({ type: 'offer', sdp: getSourceSdp() @@ -163,7 +163,7 @@ describe('TPCUtils', () => { it('sort ssrcs in case there is a single FID group', () => { const pc = new MockPeerConnection(); - const tpcUtils = new TPCUtils(pc, { }); + const tpcUtils = new TPCUtils(pc); const source = new RTCSessionDescription({ type: 'offer', sdp: getSourceSdp() From 2cf54168f8bc7776f2baf94d152f412604064245 Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Thu, 21 Nov 2024 14:20:30 -0500 Subject: [PATCH 5/6] ref(TPC) Pass the transform object along for SDP munging. --- modules/RTC/TPCUtils.js | 159 +++++++++++++++---------- modules/RTC/TraceablePeerConnection.js | 74 ++---------- 2 files changed, 110 insertions(+), 123 deletions(-) diff --git a/modules/RTC/TPCUtils.js b/modules/RTC/TPCUtils.js index 68f4febd25..2dba0e76a3 100644 --- a/modules/RTC/TPCUtils.js +++ b/modules/RTC/TPCUtils.js @@ -7,6 +7,7 @@ import { MediaDirection } from '../../service/RTC/MediaDirection'; import { MediaType } from '../../service/RTC/MediaType'; import { SIM_LAYERS, + SSRC_GROUP_SEMANTICS, STANDARD_CODEC_SETTINGS, VIDEO_QUALITY_LEVELS, VIDEO_QUALITY_SETTINGS @@ -438,12 +439,11 @@ export class TPCUtils { } /** - * Ensures that the ssrcs associated with a FID ssrc-group appear in the correct order, i.e., - * the primary ssrc first and the secondary rtx ssrc later. This is important for unified - * plan since we have only one FID group per media description. - * @param {Object} description the webRTC session description instance for the remote - * description. - * @private + * Ensures that the ssrcs associated with a FID ssrc-group appear in the correct order, i.e., the primary ssrc + * first and the secondary rtx ssrc later. This is important for unified plan since we have only one FID group per + * media description. + * @param {Object} description the webRTC session description instance for the remote description. + * @returns {Object} the modified webRTC session description instance. */ ensureCorrectOrderOfSsrcs(description) { const parsedSdp = transform.parse(description.sdp); @@ -493,14 +493,15 @@ export class TPCUtils { if (this.pc.usesCodecSelectionAPI() && rtpSender) { const { codecs } = rtpSender.getParameters(); - return codecs[0].mimeType.split('/')[1].toLowerCase(); + if (codecs?.length) { + return codecs[0].mimeType.split('/')[1].toLowerCase(); + } } const sdp = this.pc.remoteDescription?.sdp; - const defaultCodec = CodecMimeType.VP8; if (!sdp) { - return defaultCodec; + return CodecMimeType.VP8; } const parsedSdp = transform.parse(sdp); const mLine = parsedSdp.media @@ -512,17 +513,17 @@ export class TPCUtils { return Object.values(CodecMimeType).find(value => value === codec.toLowerCase()); } - return defaultCodec; + return CodecMimeType.VP8; } /** * Returns the codecs in the current order of preference as configured on the peerconnection. * - * @param {RTCSessionDescription} - The local description to be used. + * @param {string} - The local SDP to be used. * @returns {Array} */ - getConfiguredVideoCodecs(description) { - const currentSdp = description?.sdp ?? this.pc.localDescription?.sdp; + getConfiguredVideoCodecs(sdp) { + const currentSdp = sdp ?? this.pc.localDescription?.sdp; if (!currentSdp) { return []; @@ -574,6 +575,57 @@ export class TPCUtils { } ]; } + /** + * Injects a 'SIM' ssrc-group line for simulcast into the given session description object to make Jicofo happy. + * This is needed only for Firefox since it does not generate it when simulcast is enabled. + * + * @param desc A session description object (with 'type' and 'sdp' fields) + * @return A session description object with its sdp field modified to contain an inject ssrc-group for simulcast. + */ + injectSsrcGroupForUnifiedSimulcast(desc) { + const sdp = transform.parse(desc.sdp); + const video = sdp.media.find(mline => mline.type === 'video'); + + // Check if the browser supports RTX, add only the primary ssrcs to the SIM group if that is the case. + video.ssrcGroups = video.ssrcGroups || []; + const fidGroups = video.ssrcGroups.filter(group => group.semantics === SSRC_GROUP_SEMANTICS.FID); + + if (video.simulcast || video.simulcast_03) { + const ssrcs = []; + + if (fidGroups && fidGroups.length) { + fidGroups.forEach(group => { + ssrcs.push(group.ssrcs.split(' ')[0]); + }); + } else { + video.ssrcs.forEach(ssrc => { + if (ssrc.attribute === 'msid') { + ssrcs.push(ssrc.id); + } + }); + } + if (video.ssrcGroups.find(group => group.semantics === SSRC_GROUP_SEMANTICS.SIM)) { + // Group already exists, no need to do anything + return desc; + } + + // Add a SIM group for every 3 FID groups. + for (let i = 0; i < ssrcs.length; i += 3) { + const simSsrcs = ssrcs.slice(i, i + 3); + + video.ssrcGroups.push({ + semantics: SSRC_GROUP_SEMANTICS.SIM, + ssrcs: simSsrcs.join(' ') + }); + } + } + + return { + type: desc.type, + sdp: transform.write(sdp) + }; + } + /** * Takes in a *unified plan* offer and inserts the appropriate parameters for adding simulcast receive support. * @param {Object} desc - A session description object @@ -659,26 +711,22 @@ export class TPCUtils { /** * Munges the session description to ensure that the codec order is as per the preferred codec settings. * - * @param {RTCSessionDescription} description - the local/remote description to be munged. - * @returns {RTCSessionDescription} - the munged local/remote description. + * @param {transform.SessionDescription} parsedSdp that needs to be munged + * @returns {transform.SessionDescription} the munged SDP. */ - mungeCodecOrder(description) { + mungeCodecOrder(parsedSdp) { const codecSettings = this.pc.codecSettings; if (!codecSettings) { - return description; + return parsedSdp; } + const mungedSdp = parsedSdp; const { isP2P } = this.options; - const parsedSdp = transform.parse(description.sdp); - const mLines = parsedSdp.media.filter(m => m.type === codecSettings.mediaType); - - if (!mLines.length) { - return description; - } + const mLines = mungedSdp.media.filter(m => m.type === codecSettings.mediaType); for (const mLine of mLines) { - const currentCodecs = this.getConfiguredVideoCodecs(description); + const currentCodecs = this.getConfiguredVideoCodecs(transform.write(parsedSdp)); for (const codec of currentCodecs) { if (isP2P) { @@ -707,28 +755,25 @@ export class TPCUtils { } } - return { - type: description.type, - sdp: transform.write(parsedSdp) - }; + return mungedSdp; } /** * Munges the stereo flag as well as the opusMaxAverageBitrate in the SDP, based on values set through config.js, * if present. * - * @param {RTCSessionDescription} description that needs to be munged. - * @returns {RTCSessionDescription} the munged description. + * @param {transform.SessionDescription} parsedSdp that needs to be munged. + * @returns {transform.SessionDescription} the munged SDP. */ - mungeOpus(description) { + mungeOpus(parsedSdp) { const { audioQuality } = this.options; if (!audioQuality?.enableOpusDtx && !audioQuality?.stereo && !audioQuality?.opusMaxAverageBitrate) { - return description; + return parsedSdp; } - const parsedSdp = transform.parse(description.sdp); - const mLines = parsedSdp.media.filter(m => m.type === MediaType.AUDIO); + const mungedSdp = parsedSdp; + const mLines = mungedSdp.media.filter(m => m.type === MediaType.AUDIO); for (const mLine of mLines) { const { payload } = mLine.rtp.find(protocol => protocol.codec === CodecMimeType.OPUS); @@ -780,30 +825,27 @@ export class TPCUtils { fmtpOpus.config = mungedConfig.trim(); } - return { - type: description.type, - sdp: transform.write(parsedSdp) - }; + return mungedSdp; } /** - * Munges the session description by setting the max bitrates on the video m-lines when VP9 K-SVC codec is in use. + * Munges the session SDP by setting the max bitrates on the video m-lines when VP9 K-SVC codec is in use. * - * @param {RTCSessionDescription} description - The local/remote description that needs to be munged. + * @param {transform.SessionDescription} parsedSdp that needs to be munged. * @param {boolean} isLocalSdp - Whether the max bitrate (via b=AS line in SDP) is set on local SDP. - * @returns {RTCSessionDescription} - The munged local/remote description. + * @returns {transform.SessionDescription} The munged SDP. */ - setMaxBitrates(description, isLocalSdp = false) { + setMaxBitrates(parsedSdp, isLocalSdp = false) { const pcCodecSettings = this.pc.codecSettings; if (!pcCodecSettings) { - return description; + return parsedSdp; } - const parsedSdp = transform.parse(description.sdp); // Find all the m-lines associated with the local sources. + const mungedSdp = parsedSdp; const direction = isLocalSdp ? MediaDirection.RECVONLY : MediaDirection.SENDONLY; - const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO && m.direction !== direction); + const mLines = mungedSdp.media.filter(m => m.type === MediaType.VIDEO && m.direction !== direction); const currentCodec = pcCodecSettings.codecList[0]; const codecScalabilityModeSettings = this.codecSettings[currentCodec]; @@ -846,26 +888,22 @@ export class TPCUtils { } } - return { - type: description.type, - sdp: transform.write(parsedSdp) - }; + return mungedSdp; } /** * Checks if the AV1 Dependency descriptors are negotiated on the bridge peerconnection and removes them from the - * description when codec selected is VP8 or VP9. + * SDP when codec selected is VP8 or VP9. * - * @param {RTCSessionDescription} description that needs to be munged. - * @returns {RTCSessionDescription} the munged description. + * @param {transform.SessionDescription} parsedSdp that needs to be munged. + * @returns {string} the munged SDP. */ - updateAv1DdHeaders(description) { - const parsedSdp = transform.parse(description.sdp); - const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO); - - if (!mLines.length || !browser.supportsDDExtHeaders()) { - return description; + updateAv1DdHeaders(parsedSdp) { + if (!this.supportsDDHeaderExt) { + return parsedSdp; } + const mungedSdp = parsedSdp; + const mLines = mungedSdp.media.filter(m => m.type === MediaType.VIDEO); mLines.forEach((mLine, idx) => { const senderMids = Array.from(this.pc.localTrackTransceiverMids.values()); @@ -896,9 +934,6 @@ export class TPCUtils { } }); - return { - type: description.type, - sdp: transform.write(parsedSdp) - }; + return mungedSdp; } } diff --git a/modules/RTC/TraceablePeerConnection.js b/modules/RTC/TraceablePeerConnection.js index 4f95e91c08..f0740b516d 100644 --- a/modules/RTC/TraceablePeerConnection.js +++ b/modules/RTC/TraceablePeerConnection.js @@ -1187,58 +1187,6 @@ TraceablePeerConnection.prototype.getLocalSSRC = function(localTrack) { return ssrcInfo && ssrcInfo.ssrcs[0]; }; -/** - * When doing unified plan simulcast, we'll have a set of ssrcs but no ssrc-groups on Firefox. Unfortunately, Jicofo - * will complain if it sees ssrcs with matching msids but no ssrc-group, so a ssrc-group line is injected to make - * Jicofo happy. - * - * @param desc A session description object (with 'type' and 'sdp' fields) - * @return A session description object with its sdp field modified to contain an inject ssrc-group for simulcast. - */ -TraceablePeerConnection.prototype._injectSsrcGroupForUnifiedSimulcast = function(desc) { - const sdp = transform.parse(desc.sdp); - const video = sdp.media.find(mline => mline.type === 'video'); - - // Check if the browser supports RTX, add only the primary ssrcs to the SIM group if that is the case. - video.ssrcGroups = video.ssrcGroups || []; - const fidGroups = video.ssrcGroups.filter(group => group.semantics === SSRC_GROUP_SEMANTICS.FID); - - if (video.simulcast || video.simulcast_03) { - const ssrcs = []; - - if (fidGroups && fidGroups.length) { - fidGroups.forEach(group => { - ssrcs.push(group.ssrcs.split(' ')[0]); - }); - } else { - video.ssrcs.forEach(ssrc => { - if (ssrc.attribute === 'msid') { - ssrcs.push(ssrc.id); - } - }); - } - if (video.ssrcGroups.find(group => group.semantics === SSRC_GROUP_SEMANTICS.SIM)) { - // Group already exists, no need to do anything - return desc; - } - - // Add a SIM group for every 3 FID groups. - for (let i = 0; i < ssrcs.length; i += 3) { - const simSsrcs = ssrcs.slice(i, i + 3); - - video.ssrcGroups.push({ - semantics: SSRC_GROUP_SEMANTICS.SIM, - ssrcs: simSsrcs.join(' ') - }); - } - } - - return { - type: desc.type, - sdp: transform.write(sdp) - }; -}; - /* eslint-disable-next-line vars-on-top */ const getters = { signalingState() { @@ -1261,9 +1209,8 @@ const getters = { this.trace('getLocalDescription::preTransform', dumpSDP(desc)); - // For a jvb connection, transform the SDP to Plan B first. if (!this.isP2P) { - desc = this._injectSsrcGroupForUnifiedSimulcast(desc); + desc = this.tpcUtils.injectSsrcGroupForUnifiedSimulcast(desc); this.trace('getLocalDescription::postTransform (inject ssrc group)', dumpSDP(desc)); } @@ -1457,7 +1404,7 @@ TraceablePeerConnection.prototype._assertTrackBelongs = function( * @returns {Array} */ TraceablePeerConnection.prototype.getConfiguredVideoCodecs = function(description) { - return this.tpcUtils.getConfiguredVideoCodecs(description); + return this.tpcUtils.getConfiguredVideoCodecs(description?.sdp); }; /** @@ -2001,13 +1948,18 @@ TraceablePeerConnection.prototype._setEncodings = function(localTrack) { * @returns {RTCSessionDescription} - The munged description. */ TraceablePeerConnection.prototype._mungeDescription = function(description) { - let mungedDescription = description; - this.trace('RTCSessionDescription::preTransform', dumpSDP(description)); - mungedDescription = this.tpcUtils.mungeOpus(description); - mungedDescription = this.tpcUtils.mungeCodecOrder(mungedDescription); - mungedDescription = this.tpcUtils.setMaxBitrates(mungedDescription, true); - mungedDescription = this.tpcUtils.updateAv1DdHeaders(mungedDescription); + let mungedSdp = transform.parse(description.sdp); + + mungedSdp = this.tpcUtils.mungeOpus(mungedSdp); + mungedSdp = this.tpcUtils.mungeCodecOrder(mungedSdp); + mungedSdp = this.tpcUtils.setMaxBitrates(mungedSdp, true); + mungedSdp = this.tpcUtils.updateAv1DdHeaders(mungedSdp); + const mungedDescription = { + type: description.type, + sdp: transform.write(mungedSdp) + }; + this.trace('RTCSessionDescription::postTransform', dumpSDP(mungedDescription)); return mungedDescription; From fe8b8859c6fd01179041911bde35729e77e1df65 Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Mon, 25 Nov 2024 11:19:06 -0500 Subject: [PATCH 6/6] ref(TPCUtils) Fix JSDoc and change impl of a method. --- modules/RTC/TPCUtils.js | 44 ++++++++++++++++++-------- modules/RTC/TPCUtils.spec.js | 2 +- modules/RTC/TraceablePeerConnection.js | 2 +- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/modules/RTC/TPCUtils.js b/modules/RTC/TPCUtils.js index 2dba0e76a3..d6271df1a0 100644 --- a/modules/RTC/TPCUtils.js +++ b/modules/RTC/TPCUtils.js @@ -101,6 +101,7 @@ export class TPCUtils { * @param {CodecMimeType} codec - The video codec. * @param {number} newHeight - The resolution that needs to be configured for the local video track. * @returns {Object} configuration. + * @private */ _calculateActiveEncodingParams(localVideoTrack, codec, newHeight) { const codecBitrates = this.codecSettings[codec].maxBitratesVideo; @@ -159,6 +160,22 @@ export class TPCUtils { return config; } + /** + * Returns the codecs in the current order of preference in the SDP provided. + * + * @param {transform.SessionDescription} parsedSdp the parsed SDP object. + * @returns {Array} + * @private + */ + _getConfiguredVideoCodecsImpl(parsedSdp) { + const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO); + const codecs = new Set(mLine.rtp + .filter(pt => pt.codec.toLowerCase() !== 'rtx') + .map(pt => pt.codec.toLowerCase())); + + return Array.from(codecs); + } + /** * The startup configuration for the stream encodings that are applicable to the video stream when a new sender is * created on the peerconnection. The initial config takes into account the differences in browser's simulcast @@ -172,8 +189,10 @@ export class TPCUtils { * scaleResolutionDownBy - the factor by which the encoding is scaled down from the original resolution of the * captured video. * - * @param {JitsiLocalTrack} localTrack - * @param {String} codec + * @param {JitsiLocalTrack} localTrack - The local video track. + * @param {String} codec - The codec currently in use. + * @returns {Array} - The initial configuration for the stream encodings. + * @private */ _getVideoStreamEncodings(localTrack, codec) { const captureResolution = localTrack.getCaptureResolution(); @@ -268,8 +287,9 @@ export class TPCUtils { * Returns a boolean indicating whether the video encoder is running in full SVC mode, i.e., it sends only one * video stream that has both temporal and spatial scalability. * - * @param {CodecMimeType} codec - * @returns boolean + * @param {CodecMimeType} codec - The video codec in use. + * @returns boolean - true if the video encoder is running in full SVC mode, false otherwise. + * @private */ _isRunningInFullSvcMode(codec) { return (codec === CodecMimeType.VP9 || codec === CodecMimeType.AV1) @@ -286,7 +306,8 @@ export class TPCUtils { * desktop stream at all if only the high resolution stream is enabled. * * @param {JitsiLocalTrack} localVideoTrack - The local video track. - * @returns {boolean} + * @returns {boolean} - true if the bitrate needs to be capped for the screenshare track, false otherwise. + * @private */ _isScreenshareBitrateCapped(localVideoTrack) { return localVideoTrack.getVideoType() === VideoType.DESKTOP @@ -529,12 +550,8 @@ export class TPCUtils { return []; } const parsedSdp = transform.parse(currentSdp); - const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO); - const codecs = new Set(mLine.rtp - .filter(pt => pt.codec.toLowerCase() !== 'rtx') - .map(pt => pt.codec.toLowerCase())); - return Array.from(codecs); + return this._getConfiguredVideoCodecsImpl(parsedSdp); } /** @@ -577,12 +594,13 @@ export class TPCUtils { /** * Injects a 'SIM' ssrc-group line for simulcast into the given session description object to make Jicofo happy. - * This is needed only for Firefox since it does not generate it when simulcast is enabled. + * This is needed only for Firefox since it does not generate it when simulcast is enabled but we run the check + * on all browsers just in case as it would break the functionality otherwise. * * @param desc A session description object (with 'type' and 'sdp' fields) * @return A session description object with its sdp field modified to contain an inject ssrc-group for simulcast. */ - injectSsrcGroupForUnifiedSimulcast(desc) { + injectSsrcGroupForSimulcast(desc) { const sdp = transform.parse(desc.sdp); const video = sdp.media.find(mline => mline.type === 'video'); @@ -726,7 +744,7 @@ export class TPCUtils { const mLines = mungedSdp.media.filter(m => m.type === codecSettings.mediaType); for (const mLine of mLines) { - const currentCodecs = this.getConfiguredVideoCodecs(transform.write(parsedSdp)); + const currentCodecs = this._getConfiguredVideoCodecsImpl(mungedSdp); for (const codec of currentCodecs) { if (isP2P) { diff --git a/modules/RTC/TPCUtils.spec.js b/modules/RTC/TPCUtils.spec.js index 0231557917..6b29f475fa 100644 --- a/modules/RTC/TPCUtils.spec.js +++ b/modules/RTC/TPCUtils.spec.js @@ -35,7 +35,7 @@ describe('TPCUtils', () => { it('sort ssrcs associated with all FID ssrc-groups', () => { const pc = new MockPeerConnection(); - const tpcUtils = new TPCUtils(pc, { }); + const tpcUtils = new TPCUtils(pc); const source = new RTCSessionDescription({ type: 'offer', sdp: getSourceSdp() diff --git a/modules/RTC/TraceablePeerConnection.js b/modules/RTC/TraceablePeerConnection.js index f0740b516d..818a3710e1 100644 --- a/modules/RTC/TraceablePeerConnection.js +++ b/modules/RTC/TraceablePeerConnection.js @@ -1210,7 +1210,7 @@ const getters = { this.trace('getLocalDescription::preTransform', dumpSDP(desc)); if (!this.isP2P) { - desc = this.tpcUtils.injectSsrcGroupForUnifiedSimulcast(desc); + desc = this.tpcUtils.injectSsrcGroupForSimulcast(desc); this.trace('getLocalDescription::postTransform (inject ssrc group)', dumpSDP(desc)); }