From 16b014d6810d6b42743b07e62f70b0ee67de88e2 Mon Sep 17 00:00:00 2001 From: Dono Date: Tue, 6 Sep 2022 10:53:05 +0200 Subject: [PATCH 1/6] 3 - peer: allow user to remove tracks from calls --- core/client/peer.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/client/peer.js b/core/client/peer.js index 3467eb27..3cdec1f8 100644 --- a/core/client/peer.js +++ b/core/client/peer.js @@ -227,12 +227,16 @@ peer = { const senders = call.peerConnection.getSenders(); const existingSenderAudioTrack = senders.find(sender => sender.track.kind === 'audio'); - if (existingSenderAudioTrack) existingSenderAudioTrack.replaceTrack(audioTrack); - else call.peerConnection.addTrack(audioTrack); + if (existingSenderAudioTrack) { + if (audioTrack) existingSenderAudioTrack.replaceTrack(audioTrack); + else call.peerConnection.removeTrack(existingSenderAudioTrack); + } else if (audioTrack) call.peerConnection.addTrack(audioTrack); const existingSenderVideoTrack = senders.find(sender => sender.track.kind === 'video'); - if (existingSenderVideoTrack) existingSenderVideoTrack.replaceTrack(videoTrack); - else call.peerConnection.addTrack(videoTrack); + if (existingSenderVideoTrack) { + if (videoTrack) existingSenderVideoTrack.replaceTrack(videoTrack); + else call.peerConnection.removeTrack(existingSenderVideoTrack); + } else if (videoTrack) call.peerConnection.addTrack(videoTrack); if (!existingSenderAudioTrack || !existingSenderVideoTrack) debug(`updatePeersStream: stream main track added for user`, { key }); else debug(`updatePeersStream: stream main track updated for user`, { key }); From 778d989f247ff8df7a6899912a88d12328f3a976 Mon Sep 17 00:00:00 2001 From: Dono Date: Tue, 6 Sep 2022 10:56:35 +0200 Subject: [PATCH 2/6] 1 - peer: stop using underscore in updatePeersStream --- core/client/peer.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/client/peer.js b/core/client/peer.js index 3cdec1f8..accd077c 100644 --- a/core/client/peer.js +++ b/core/client/peer.js @@ -216,13 +216,15 @@ peer = { updatePeersStream(stream, type) { debug('updatePeersStream: start', { stream, type }); + const callEntries = Object.entries(this.calls); + if (type === streamTypes.main) { debug(`updatePeersStream: main stream ${stream.id}`, { stream }); const audioTrack = stream.getAudioTracks()[0]; const videoTrack = stream.getVideoTracks()[0]; // note: to add a track it is necessary to renegotiate the connection with the remote user (https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addTrack) - _.each(this.calls, (call, key) => { + callEntries.forEach(([key, call]) => { if (key.indexOf('-screen') !== -1) return; const senders = call.peerConnection.getSenders(); @@ -245,7 +247,7 @@ peer = { debug(`updatePeersStream: screen share stream ${stream.id}`, { stream }); const screenTrack = stream.getVideoTracks()[0]; - _.each(this.calls, (call, key) => { + callEntries.forEach(([key, call]) => { if (key.indexOf('-screen') === -1) return; const senders = call.peerConnection.getSenders(); let trackUpdated = false; From 95ab5264959e376974230bcd2c240bf3487ef03a Mon Sep 17 00:00:00 2001 From: Dono Date: Tue, 6 Sep 2022 16:10:47 +0200 Subject: [PATCH 3/6] 15 - peer: split streams from peers + add methods to close peer calls --- core/client/lemverse.js | 13 ++-- core/client/peer.js | 140 ++++++++++++++++++------------------ core/client/user-streams.js | 14 ++-- 3 files changed, 82 insertions(+), 85 deletions(-) diff --git a/core/client/lemverse.js b/core/client/lemverse.js index daa35c27..7e483b26 100644 --- a/core/client/lemverse.js +++ b/core/client/lemverse.js @@ -177,12 +177,13 @@ Template.lemverse.onCreated(function () { if (!user) return; Tracker.nonreactive(() => { if (userProximitySensor.nearUsersCount() === 0) userStreams.destroyStream(streamTypes.main); - else if (!user.profile.shareVideo) userStreams.video(false); - else if (user.profile.shareVideo) { + else if (!user.profile.shareVideo) { + userStreams.video(false); + } else if (user.profile.shareVideo) { const forceNewStream = userStreams.shouldCreateNewStream(streamTypes.main, true, true); userStreams.createStream(forceNewStream).then(() => { userStreams.video(true); - userProximitySensor.callProximityStartedForAllNearUsers(); + peer.call(Object.values(userProximitySensor.nearUsers)); }); } @@ -201,9 +202,13 @@ Template.lemverse.onCreated(function () { if (user.profile.shareScreen) meet.shareScreen(); else meet.unshareScreen(); } else if (user.profile.shareScreen) { - userStreams.createScreenStream().then(() => userStreams.screen(true)); + userStreams.createScreenStream().then(() => { + userStreams.screen(true); + peer.call(Object.values(userProximitySensor.nearUsers)); + }); } else { userStreams.screen(false); + peer.closePeerCalls(false, streamTypes.screen); } }); }); diff --git a/core/client/peer.js b/core/client/peer.js index accd077c..49dca7ca 100644 --- a/core/client/peer.js +++ b/core/client/peer.js @@ -83,27 +83,13 @@ peer = { closeCall(userId, origin) { debug(`closeCall: start (${origin})`, { userId }); - let activeCallsCount = 0; - const _close = (remote, user, type) => { - const callsSource = remote ? this.remoteCalls : this.calls; - const call = callsSource[`${user}-${type}`]; - if (call) { - activeCallsCount++; - call.close(); - } - - delete callsSource[`${user}-${type}`]; - }; - this.unlockCall(userId, true); - _close(false, userId, streamTypes.main); - _close(false, userId, streamTypes.screen); - _close(true, userId, streamTypes.main); - _close(true, userId, streamTypes.screen); this.cancelWaitingCallAction(userId); - const debutText = activeCallsCount ? 'closeCall: call was active' : 'closeCall: call was inactive'; - debug(debutText, { sourceAmount: activeCallsCount }); + let closedPeerCallsCount = 0; + closedPeerCallsCount += this._closeUserPeerCalls(true, userId); + closedPeerCallsCount += this._closeUserPeerCalls(false, userId); + if (closedPeerCallsCount) audioManager.play('webrtc-out.mp3', 0.2); let streamsByUsers = this.remoteStreamsByUsers.get(); streamsByUsers.map(usr => { @@ -132,10 +118,6 @@ peer = { $(`.js-video-${userId}-user`).remove(); debug('closeCall: call closed successfully', { userId }); - - if (!activeCallsCount) return; - - audioManager.play('webrtc-out.mp3', 0.2); }, close(userId, timeout = 0, origin = null) { @@ -155,6 +137,35 @@ peer = { }; }, + closePeerCalls(remote, type) { + const callsSource = remote ? this.remoteCalls : this.calls; + const callEntries = Object.entries(callsSource); + const typeKey = `-${type}`; + + callEntries.forEach(([key, call]) => { + if (key.indexOf(typeKey) === -1) return; + + call.close(); + delete callsSource[key]; + }); + }, + + _closeUserPeerCalls(remote, userId) { + const callsSource = remote ? this.remoteCalls : this.calls; + const callEntries = Object.entries(callsSource); + let closedCount = 0; + + callEntries.forEach(([key, call]) => { + if (key.indexOf(userId) === -1) return; + + call.close(); + delete callsSource[key]; + closedCount++; + }); + + return closedCount; + }, + createPeerCall(peer, user, stream, streamType) { debug(`createPeerCall: calling remote user`, { user: user._id, streamType }); if (!stream) { error(`createPeerCall: stream is undefined`, { user, stream }); return; } @@ -213,63 +224,42 @@ peer = { delete this.peerInstance; }, + /** + * To add a track it is necessary to renegotiate the connection with the remote user. + * @see https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addTrack) + */ updatePeersStream(stream, type) { debug('updatePeersStream: start', { stream, type }); - const callEntries = Object.entries(this.calls); - - if (type === streamTypes.main) { - debug(`updatePeersStream: main stream ${stream.id}`, { stream }); - const audioTrack = stream.getAudioTracks()[0]; - const videoTrack = stream.getVideoTracks()[0]; - - // note: to add a track it is necessary to renegotiate the connection with the remote user (https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addTrack) - callEntries.forEach(([key, call]) => { - if (key.indexOf('-screen') !== -1) return; - const senders = call.peerConnection.getSenders(); - - const existingSenderAudioTrack = senders.find(sender => sender.track.kind === 'audio'); - if (existingSenderAudioTrack) { - if (audioTrack) existingSenderAudioTrack.replaceTrack(audioTrack); - else call.peerConnection.removeTrack(existingSenderAudioTrack); - } else if (audioTrack) call.peerConnection.addTrack(audioTrack); - - const existingSenderVideoTrack = senders.find(sender => sender.track.kind === 'video'); - if (existingSenderVideoTrack) { - if (videoTrack) existingSenderVideoTrack.replaceTrack(videoTrack); - else call.peerConnection.removeTrack(existingSenderVideoTrack); - } else if (videoTrack) call.peerConnection.addTrack(videoTrack); - - if (!existingSenderAudioTrack || !existingSenderVideoTrack) debug(`updatePeersStream: stream main track added for user`, { key }); - else debug(`updatePeersStream: stream main track updated for user`, { key }); - }); - } else if (type === streamTypes.screen) { - debug(`updatePeersStream: screen share stream ${stream.id}`, { stream }); - const screenTrack = stream.getVideoTracks()[0]; - - callEntries.forEach(([key, call]) => { - if (key.indexOf('-screen') === -1) return; - const senders = call.peerConnection.getSenders(); - let trackUpdated = false; - - senders.forEach(sender => { - if (sender.track.id === screenTrack.id || sender.track.kind !== 'video') return; - sender.replaceTrack(screenTrack); - trackUpdated = true; - }); - - if (trackUpdated) debug(`updatePeersStream: stream main track updated for user ${key}`); - }); - } + const typeKey = `-${streamTypes.screen}`; + const audioTrack = stream.getAudioTracks()[0]; + const videoTrack = stream.getVideoTracks()[0]; + + Object.entries(this.calls).forEach(([key, call]) => { + if (key.indexOf(typeKey) === -1) return; + const senders = call.peerConnection.getSenders(); + + const existingSenderAudioTrack = senders.find(sender => sender.track.kind === 'audio'); + if (existingSenderAudioTrack) { + if (audioTrack) existingSenderAudioTrack.replaceTrack(audioTrack); + else call.peerConnection.removeTrack(existingSenderAudioTrack); + } else if (audioTrack) call.peerConnection.addTrack(audioTrack); + + const existingSenderVideoTrack = senders.find(sender => sender.track.kind === 'video'); + if (existingSenderVideoTrack) { + if (videoTrack) existingSenderVideoTrack.replaceTrack(videoTrack); + else call.peerConnection.removeTrack(existingSenderVideoTrack); + } else if (videoTrack) call.peerConnection.addTrack(videoTrack); + }); }, - onProximityStarted(nearUsers) { + call(users) { if (!this.isEnabled()) return; const user = Meteor.user(); if (user?.profile.guest) return; // disable proximity sensor for guest user - nearUsers.forEach(nearUser => { + users.forEach(nearUser => { if (this.isCallInState(nearUser._id, callAction.open)) return; this.cancelWaitingCallAction(nearUser._id); @@ -289,13 +279,21 @@ peer = { }); }, - onProximityEnded(users) { + hangUp(users, origin = 'hang-up') { users.forEach(user => { if (this.lockedCalls[user._id]) return; - this.close(user._id, Meteor.settings.public.peer.delayBeforeClosingCall, 'proximity-ended'); + this.close(user._id, Meteor.settings.public.peer.delayBeforeClosingCall, origin); }); }, + onProximityStarted(nearUsers) { + this.call(nearUsers); + }, + + onProximityEnded(users) { + this.hangUp(users, 'proximity-ended'); + }, + isCallInState(userId, state) { return this.waitingCallActions[userId]?.action === state; }, diff --git a/core/client/user-streams.js b/core/client/user-streams.js index a988d6db..e301fa96 100644 --- a/core/client/user-streams.js +++ b/core/client/user-streams.js @@ -36,16 +36,10 @@ userStreams = { screen(enabled) { const { instance: screenStream } = this.streams.screen; - if (screenStream && !enabled) { - this.stopTracks(screenStream); - this.streams.screen.instance = undefined; - _.each(peer.calls, (call, key) => { - if (key.indexOf('-screen') === -1) return; - if (Meteor.user().options?.debug) log('me -> you screen ****** I stop sharing screen, call closing', key); - call.close(); - delete peer.calls[key]; - }); - } else if (enabled) userProximitySensor.callProximityStartedForAllNearUsers(); + if (!screenStream || enabled) return; + + this.stopTracks(screenStream); + this.streams.screen.instance = undefined; }, destroyStream(type) { From f7db58b107176db433a94da0074b854f3b4af5c5 Mon Sep 17 00:00:00 2001 From: Dono Date: Wed, 7 Sep 2022 14:54:42 +0200 Subject: [PATCH 4/6] 0 - peer: remove useless condition --- core/client/lemverse.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/client/lemverse.js b/core/client/lemverse.js index 7e483b26..0cf261f0 100644 --- a/core/client/lemverse.js +++ b/core/client/lemverse.js @@ -158,7 +158,7 @@ Template.lemverse.onCreated(function () { Tracker.nonreactive(() => { if (userProximitySensor.nearUsersCount() === 0) userStreams.destroyStream(streamTypes.main); else if (!user.profile.shareAudio) userStreams.audio(false); - else if (user.profile.shareAudio) { + else { userStreams.createStream().then(() => { userStreams.audio(true); userProximitySensor.callProximityStartedForAllNearUsers(); @@ -179,7 +179,7 @@ Template.lemverse.onCreated(function () { if (userProximitySensor.nearUsersCount() === 0) userStreams.destroyStream(streamTypes.main); else if (!user.profile.shareVideo) { userStreams.video(false); - } else if (user.profile.shareVideo) { + } else { const forceNewStream = userStreams.shouldCreateNewStream(streamTypes.main, true, true); userStreams.createStream(forceNewStream).then(() => { userStreams.video(true); From 769d70a81a0babf281d4581d8f4c597c8425a965 Mon Sep 17 00:00:00 2001 From: Dono Date: Wed, 7 Sep 2022 14:58:32 +0200 Subject: [PATCH 5/6] 0 - streams: remove useless log --- core/client/user-streams.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/core/client/user-streams.js b/core/client/user-streams.js index e301fa96..800e363c 100644 --- a/core/client/user-streams.js +++ b/core/client/user-streams.js @@ -43,16 +43,13 @@ userStreams = { }, destroyStream(type) { - const debug = Meteor.user()?.options?.debug; const { instance: stream } = type === streamTypes.main ? this.streams.main : this.streams.screen; + if (!stream) return; + + const debug = Meteor.user({ fields: { 'options.debug': 1 } })?.options?.debug; if (debug) log('destroyStream: start', { stream, type }); - if (!stream) { - if (debug) log('destroyStream: cancelled (stream was not alive)'); - return; - } this.stopTracks(stream); - if (stream === this.streams.main.instance) this.streams.main.instance = undefined; else if (stream === this.streams.screen.instance) this.streams.screen.instance = undefined; }, From 6d245e86e36eb23239d3a766346932416fd02ca1 Mon Sep 17 00:00:00 2001 From: Dono Date: Wed, 7 Sep 2022 15:54:43 +0200 Subject: [PATCH 6/6] 2 - peer: move remoteStreams update logic to a function + remove useless jquery selector --- core/client/peer.js | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/core/client/peer.js b/core/client/peer.js index 49dca7ca..8047b5c7 100644 --- a/core/client/peer.js +++ b/core/client/peer.js @@ -80,17 +80,7 @@ peer = { _.each(this.calls, call => this.close(call.peer, Meteor.settings.public.peer.delayBeforeClosingCall, 'close-all')); }, - closeCall(userId, origin) { - debug(`closeCall: start (${origin})`, { userId }); - - this.unlockCall(userId, true); - this.cancelWaitingCallAction(userId); - - let closedPeerCallsCount = 0; - closedPeerCallsCount += this._closeUserPeerCalls(true, userId); - closedPeerCallsCount += this._closeUserPeerCalls(false, userId); - if (closedPeerCallsCount) audioManager.play('webrtc-out.mp3', 0.2); - + _removeUserFromRemoteStreamsList(userId) { let streamsByUsers = this.remoteStreamsByUsers.get(); streamsByUsers.map(usr => { if (usr._id === userId) { @@ -105,6 +95,20 @@ peer = { // We clean up remoteStreamsByUsers table by deleting all the users who have neither webcam or screen sharing active streamsByUsers = streamsByUsers.filter(usr => usr.main.srcObject !== undefined || usr.screen.srcObject !== undefined || usr.waitingCallAnswer); this.remoteStreamsByUsers.set(streamsByUsers); + }, + + closeCall(userId, origin) { + debug(`closeCall: start (${origin})`, { userId }); + + this.unlockCall(userId, true); + this.cancelWaitingCallAction(userId); + + let closedPeerCallsCount = 0; + closedPeerCallsCount += this._closeUserPeerCalls(true, userId); + closedPeerCallsCount += this._closeUserPeerCalls(false, userId); + if (closedPeerCallsCount) audioManager.play('webrtc-out.mp3', 0.2); + + this._removeUserFromRemoteStreamsList(userId); if (!this.hasActiveStreams()) { userStreams.destroyStream(streamTypes.main); @@ -116,7 +120,6 @@ peer = { delete this.callStartDates[userId]; } - $(`.js-video-${userId}-user`).remove(); debug('closeCall: call closed successfully', { userId }); },