diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 614301f7..8a701194 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -25,7 +25,7 @@ export class AggregatedStats { inboundAudioStats: InboundAudioStats; lastVideoStats: InboundVideoStats; lastAudioStats: InboundAudioStats; - candidatePair: CandidatePairStats; + candidatePairs: Array; DataChannelStats: DataChannelStats; localCandidates: Array; remoteCandidates: Array; @@ -37,7 +37,6 @@ export class AggregatedStats { constructor() { this.inboundVideoStats = new InboundVideoStats(); this.inboundAudioStats = new InboundAudioStats(); - this.candidatePair = new CandidatePairStats(); this.DataChannelStats = new DataChannelStats(); this.outBoundVideoStats = new OutBoundVideoStats(); this.sessionStats = new SessionStats(); @@ -52,6 +51,7 @@ export class AggregatedStats { processStats(rtcStatsReport: RTCStatsReport) { this.localCandidates = new Array(); this.remoteCandidates = new Array(); + this.candidatePairs = new Array(); rtcStatsReport.forEach((stat) => { const type: RTCStatsTypePS = stat.type; @@ -120,16 +120,10 @@ export class AggregatedStats { * @param stat - the stats coming in from ice candidates */ handleCandidatePair(stat: CandidatePairStats) { - this.candidatePair.bytesReceived = stat.bytesReceived; - this.candidatePair.bytesSent = stat.bytesSent; - this.candidatePair.localCandidateId = stat.localCandidateId; - this.candidatePair.remoteCandidateId = stat.remoteCandidateId; - this.candidatePair.nominated = stat.nominated; - this.candidatePair.readable = stat.readable; - this.candidatePair.selected = stat.selected; - this.candidatePair.writable = stat.writable; - this.candidatePair.state = stat.state; - this.candidatePair.currentRoundTripTime = stat.currentRoundTripTime; + + // Add the candidate pair to the candidate pair array + this.candidatePairs.push(stat) + } /** @@ -162,6 +156,8 @@ export class AggregatedStats { localCandidate.protocol = stat.protocol; localCandidate.candidateType = stat.candidateType; localCandidate.id = stat.id; + localCandidate.relayProtocol = stat.relayProtocol; + localCandidate.transportId = stat.transportId; this.localCandidates.push(localCandidate); } @@ -171,12 +167,14 @@ export class AggregatedStats { */ handleRemoteCandidate(stat: CandidateStat) { const RemoteCandidate = new CandidateStat(); - RemoteCandidate.label = 'local-candidate'; + RemoteCandidate.label = 'remote-candidate'; RemoteCandidate.address = stat.address; RemoteCandidate.port = stat.port; RemoteCandidate.protocol = stat.protocol; RemoteCandidate.id = stat.id; RemoteCandidate.candidateType = stat.candidateType; + RemoteCandidate.relayProtocol = stat.relayProtocol; + RemoteCandidate.transportId = stat.transportId this.remoteCandidates.push(RemoteCandidate); } @@ -308,4 +306,12 @@ export class AggregatedStats { isNumber(value: unknown): boolean { return typeof value === 'number' && isFinite(value); } + + /** + * Helper function to return the active candidate pair + * @returns The candidate pair that is currently receiving data + */ + public getActiveCandidatePair(): CandidatePairStats | null { + return this.candidatePairs.find((candidatePair) => candidatePair.bytesReceived > 0, null) + } } diff --git a/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts b/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts index 6bb938ba..39d00a87 100644 --- a/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts +++ b/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts @@ -6,12 +6,19 @@ export class CandidatePairStats { bytesReceived: number; bytesSent: number; + currentRoundTripTime: number; + id: string; + lastPacketReceivedTimestamp: number; + lastPacketSentTimestamp: number; localCandidateId: string; - remoteCandidateId: string; nominated: boolean; + priority: number; readable: boolean; - writable: boolean; + remoteCandidateId: string; selected: boolean; state: string; - currentRoundTripTime: number; + timestamp: number; + transportId: string; + type: string; + writable: boolean; } diff --git a/Frontend/library/src/PeerConnectionController/CandidateStat.ts b/Frontend/library/src/PeerConnectionController/CandidateStat.ts index d26fafae..03ceeee5 100644 --- a/Frontend/library/src/PeerConnectionController/CandidateStat.ts +++ b/Frontend/library/src/PeerConnectionController/CandidateStat.ts @@ -4,10 +4,12 @@ * ICE Candidate Stat collected from the RTC Stats Report */ export class CandidateStat { - label: string; - id: string; address: string; candidateType: string; + id: string; + label: string; port: number; protocol: 'tcp' | 'udp'; + relayProtocol: 'tcp' | 'udp' | 'tls'; + transportId: string; } diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts index 71d86c21..0df5648b 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts @@ -398,9 +398,9 @@ describe('PixelStreaming', () => { expect.objectContaining({ data: { aggregatedStats: expect.objectContaining({ - candidatePair: expect.objectContaining({ - bytesReceived: 123 - }), + candidatePairs: [ + expect.objectContaining({ bytesReceived: 123 }) + ], localCandidates: [ expect.objectContaining({ address: 'mock-address' }) ] diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index a8aca186..1b18e7c7 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -28,7 +28,8 @@ import { WebRtcSdpEvent, DataChannelLatencyTestResponseEvent, DataChannelLatencyTestResultEvent, - PlayerCountEvent + PlayerCountEvent, + WebRtcTCPRelayDetectedEvent } from '../Util/EventEmitter'; import { MessageOnScreenKeyboard } from '../WebSockets/MessageReceive'; import { WebXRController } from '../WebXR/WebXRController'; @@ -62,6 +63,7 @@ export class PixelStreaming { protected _webRtcController: WebRtcPlayerController; protected _webXrController: WebXRController; protected _dataChannelLatencyTestController: DataChannelLatencyTestController; + /** * Configuration object. You can read or modify config through this object. Whenever * the configuration is changed, the library will emit a `settingsChanged` event. @@ -116,6 +118,13 @@ export class PixelStreaming { this.onScreenKeyboardHelper.showOnScreenKeyboard(command); this._webXrController = new WebXRController(this._webRtcController); + + // Add event listener for the webRtcConnected event + this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => { + + // Bind to the stats received event + this._eventEmitter.addEventListener("statsReceived", this._setupWebRtcTCPRelayDetection.bind(this)); + }); } /** @@ -627,6 +636,28 @@ export class PixelStreaming { ); } + // Sets up to emit the webrtc tcp relay detect event + _setupWebRtcTCPRelayDetection(statsReceivedEvent: StatsReceivedEvent) { + // Get the active candidate pair + let activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair(); + + // Check if the active candidate pair is not null + if (activeCandidatePair != null) { + + // Get the local candidate assigned to the active candidate pair + let localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null) + + // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp + if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp') { + + // Send the web rtc tcp relay detected event + this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); + } + // The check is completed and the stats listen event can be removed + this._eventEmitter.removeEventListener("statsReceived", this._setupWebRtcTCPRelayDetection); + } + } + /** * Request a connection latency test. * NOTE: There are plans to refactor all request* functions. Expect changes if you use this! diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index 64efd0ca..de1d433d 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -537,6 +537,16 @@ export class PlayerCountEvent extends Event { } } +/** + * An event that is emitted when the webRTC connections is relayed over TCP. + */ +export class WebRtcTCPRelayDetectedEvent extends Event { + readonly type: 'webRtcTCPRelayDetected'; + constructor() { + super('webRtcTCPRelayDetected'); + } +} + export type PixelStreamingEvent = | AfkWarningActivateEvent | AfkWarningUpdateEvent @@ -573,7 +583,8 @@ export type PixelStreamingEvent = | XrSessionStartedEvent | XrSessionEndedEvent | XrFrameEvent - | PlayerCountEvent; + | PlayerCountEvent + | WebRtcTCPRelayDetectedEvent; export class EventEmitter extends EventTarget { /** diff --git a/Frontend/ui-library/src/Application/Application.ts b/Frontend/ui-library/src/Application/Application.ts index e17ce403..40739e48 100644 --- a/Frontend/ui-library/src/Application/Application.ts +++ b/Frontend/ui-library/src/Application/Application.ts @@ -378,6 +378,14 @@ export class Application { ({ data: { count }}) => this.onPlayerCount(count) ); + this.stream.addEventListener( + 'webRtcTCPRelayDetected', + ({}) => + Logger.Warning( + Logger.GetStackTrace(), + `Stream quailty degraded due to network enviroment, stream is relayed over TCP.` + ) + ); } /** diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index ee364c4e..e5436639 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -1,7 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. import { LatencyTest } from './LatencyTest'; -import { InitialSettings, Logger, PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4'; +import { CandidatePairStats, InitialSettings, Logger, PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4'; import { AggregatedStats } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4'; import { MathUtils } from '../Util/MathUtils'; import {DataChannelLatencyTest} from "./DataChannelLatencyTest"; @@ -318,14 +318,17 @@ export class StatsPanel { ); } + // Store the active candidate pair return a new Candidate pair stat if getActiveCandidate is null + let activeCandidatePair = stats.getActiveCandidatePair() != null ? stats.getActiveCandidatePair() : new CandidatePairStats(); + // RTT const netRTT = Object.prototype.hasOwnProperty.call( - stats.candidatePair, + activeCandidatePair, 'currentRoundTripTime' - ) && stats.isNumber(stats.candidatePair.currentRoundTripTime) + ) && stats.isNumber(activeCandidatePair.currentRoundTripTime) ? numberFormat.format( - stats.candidatePair.currentRoundTripTime * 1000 + activeCandidatePair.currentRoundTripTime * 1000 ) : "Can't calculate"; this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT);