diff --git a/static/js/directives/audiovideo.js b/static/js/directives/audiovideo.js index e752c281..dbca88b3 100644 --- a/static/js/directives/audiovideo.js +++ b/static/js/directives/audiovideo.js @@ -26,8 +26,7 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ var controller = ['$scope', '$element', '$attrs', function($scope, $element, $attrs) { - var peers = {}; - var events = $({}); + var streams = {}; $scope.container = $element.get(0); $scope.layoutparent = $element.parent(); @@ -39,6 +38,9 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ $scope.hasUsermedia = false; $scope.isActive = false; + $scope.haveStreams = false; + + $scope.peersTalking = {}; $scope.rendererName = $scope.defaultRendererName = "democrazy"; @@ -46,31 +48,30 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ $scope.addRemoteStream = function(stream, currentcall) { - //console.log("Add remote stream to scope", pc.id, stream); + if (streams.hasOwnProperty(stream.id)) { + console.warn("Cowardly refusing to add stream id twice", stream.id, currentcall); + return; + } + + //console.log("Add remote stream to scope", stream.id, stream, currentcall); // Create scope. - var subscope = $scope.$new(true); + var subscope = $scope.$new(); var peerid = subscope.peerid = currentcall.id; buddyData.push(peerid); subscope.withvideo = false; subscope.onlyaudio = false; - subscope.talking = false; subscope.destroyed = false; - subscope.applyTalking = function(talking) { - subscope.talking = !! talking; - safeApply(subscope); - }; subscope.$on("active", function() { - console.log("Stream scope is now active", peerid); - events.triggerHandler("active." + peerid, [subscope, currentcall, stream]); + console.log("Stream scope is now active", stream.id, peerid); }); subscope.$on("$destroy", function() { - console.log("Destroyed scope for audiovideo", subscope); + console.log("Destroyed scope for stream", stream.id, peerid); subscope.destroyed = true; }); - console.log("Created stream scope", peerid); + console.log("Created stream scope", stream.id, peerid); // Add created scope. - peers[peerid] = subscope; + streams[stream.id] = subscope; // Render template. peerTemplate(subscope, function(clonedElement, scope) { @@ -118,10 +119,11 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ $scope.removeRemoteStream = function(stream, currentcall) { - var subscope = peers[currentcall.id]; + //console.log("remove stream", stream, stream.id, currentcall); + var subscope = streams[stream.id]; if (subscope) { buddyData.pop(currentcall.id); - delete peers[currentcall.id]; + delete streams[stream.id]; //console.log("remove scope", subscope); if (subscope.element) { subscope.element.remove(); @@ -134,17 +136,9 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ // Talking updates receiver. mediaStream.api.e.on("received.talking", function(event, id, from, talking) { - var scope = peers[from]; - //console.log("received.talking", talking, scope); - if (scope) { - scope.applyTalking(talking); - } else { - console.log("Received talking state without scope -> adding event.", from, talking); - events.one("active." + from, function(event, scope) { - console.log("Applying previously received talking state", from, talking); - scope.applyTalking(talking); - }); - } + $scope.$apply(function(scope) { + scope.peersTalking[from] = !!talking; + }); }); $scope.$on("active", function(currentcall) { @@ -177,27 +171,36 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ mediaStream.webrtc.e.on("usermedia", function(event, usermedia) { - //console.log("XXXXXXXXXXXXXXXXXXXXXXXXX usermedia event", usermedia); - $scope.hasUsermedia = true; - usermedia.attachMediaStream($scope.localVideo); - var count = 0; - var waitForLocalVideo = function() { - if (!$scope.hasUsermedia) { - return; - } - if ($scope.localVideo.videoWidth > 0) { - $scope.localVideo.style.opacity = 1; - $scope.redraw(); - } else { - count++; - if (count < 100) { - setTimeout(waitForLocalVideo, 100); + //console.log("XXXX XXXXXXXXXXXXXXXXXXXXX usermedia event", usermedia); + if ($scope.haveStreams) { + + usermedia.attachMediaStream($scope.miniVideo); + $scope.redraw(); + + } else { + + $scope.hasUsermedia = true; + usermedia.attachMediaStream($scope.localVideo); + var count = 0; + var waitForLocalVideo = function() { + if (!$scope.hasUsermedia) { + return; + } + if ($scope.localVideo.videoWidth > 0) { + $scope.localVideo.style.opacity = 1; + $scope.redraw(); } else { - console.warn("Timeout while waiting for local video.") + count++; + if (count < 100) { + setTimeout(waitForLocalVideo, 100); + } else { + console.warn("Timeout while waiting for local video.") + } } - } - }; - waitForLocalVideo(); + }; + waitForLocalVideo(); + + } }); @@ -205,6 +208,7 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ $scope.hasUsermedia = false; $scope.isActive = false; + $scope.peersTalking = {}; if (BigScreen.enabled) { BigScreen.exit(); } @@ -220,20 +224,22 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ $scope.localVideo.style.opacity = 0; $scope.remoteVideos.style.opacity = 0; $element.removeClass('active'); - _.each(peers, function(scope, k) { + _.each(streams, function(scope, k) { scope.$destroy(); - delete peers[k]; + delete streams[k]; }); $scope.rendererName = $scope.defaultRendererName; + $scope.haveStreams = false; }); mediaStream.webrtc.e.on("streamadded", function(event, stream, currentcall) { console.log("Remote stream added.", stream, currentcall); - if (_.isEmpty(peers)) { + if (!$scope.haveStreams) { //console.log("First stream"); $window.reattachMediaStream($scope.miniVideo, $scope.localVideo); + $scope.haveStreams = true; } $scope.addRemoteStream(stream, currentcall); @@ -247,7 +253,7 @@ define(['jquery', 'underscore', 'text!partials/audiovideo.html', 'text!partials/ }); return { - peers: peers + streams: streams }; }]; diff --git a/static/js/mediastream/peercall.js b/static/js/mediastream/peercall.js index be961623..d2ee4e3e 100644 --- a/static/js/mediastream/peercall.js +++ b/static/js/mediastream/peercall.js @@ -182,7 +182,11 @@ define(['jquery', 'underscore', 'mediastream/utils', 'mediastream/peerconnection PeerCall.prototype.onRemoteStreamAdded = function(stream) { - this.streams[stream] = true; + var id = stream.id; + if (this.streams.hasOwnProperty(id)) { + return; + } + this.streams[id] = stream; this.e.triggerHandler("remoteStreamAdded", [stream, this]); }; @@ -191,7 +195,7 @@ define(['jquery', 'underscore', 'mediastream/utils', 'mediastream/peerconnection this.e.triggerHandler("remoteStreamRemoved", [stream, this]); if (stream) { - delete this.streams[stream]; + delete this.streams[stream.id]; } }; @@ -299,13 +303,18 @@ define(['jquery', 'underscore', 'mediastream/utils', 'mediastream/peerconnection datachannel.close(); }); this.datachannels = {}; - this.streams = {}; if (this.peerconnection) { this.peerconnection.close(); this.peerconnection = null; } + // Trigger event for all previously added streams. + _.each(this.streams, _.bind(function(stream, id) { + this.e.triggerHandler("remoteStreamRemoved", [stream, this]); + }, this)); + this.streams = {}; + console.log("Peercall close", this); this.e.triggerHandler("closed", [this]); diff --git a/static/js/mediastream/peerconnection.js b/static/js/mediastream/peerconnection.js index f91d2b57..3a7abd4b 100644 --- a/static/js/mediastream/peerconnection.js +++ b/static/js/mediastream/peerconnection.js @@ -72,14 +72,7 @@ define(['jquery', 'underscore', 'webrtc.adapter'], function($, _) { pc.onremovestream = _.bind(this.onRemoteStreamRemoved, this); // NOTE(longsleep): onnegotiationneeded is not supported by Firefox // https://bugzilla.mozilla.org/show_bug.cgi?id=840728 - if (webrtcDetectedBrowser === "firefox") { - window.setTimeout(_.bind(function() { - // Trigger onNegotiationNeeded once for Firefox. - this.onNegotiationNeeded({target: pc}); - }, this), 0); - } else { - pc.onnegotiationneeded = _.bind(this.onNegotiationNeeded, this); - } + pc.onnegotiationneeded = _.bind(this.onNegotiationNeeded, this); pc.ondatachannel = _.bind(this.onDatachannel, this); pc.onsignalingstatechange = function(event) { // XXX(longsleep): Remove this or handle it in a real function. @@ -253,8 +246,6 @@ define(['jquery', 'underscore', 'webrtc.adapter'], function($, _) { this.pc.close(); } - this.currentcall.onRemoteStreamRemoved(null); - this.datachannel = null; this.pc = null; diff --git a/static/js/mediastream/usermedia.js b/static/js/mediastream/usermedia.js index 2c1d9db5..6b57ebf7 100644 --- a/static/js/mediastream/usermedia.js +++ b/static/js/mediastream/usermedia.js @@ -32,9 +32,14 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ this.started = false; this.delay = 0; + this.audioMute = false; + this.videoMute = false; + this.mediaConstraints = null; + // Audio level. this.audioLevel = 0; if (!this.options.noaudio && context && context.createScriptProcessor) { + this.audioSource = null; this.audioProcessor = context.createScriptProcessor(2048, 1, 1); this.audioProcessor.onaudioprocess = _.bind(function(event) { @@ -54,6 +59,21 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ this.audioLevel = rms; //console.log("this.audioLevel", this.audioLevel); }, this); + + // Connect stream to audio processor if supported. + if (context.createMediaStreamSource) { + this.e.bind("localstream", _.bind(function(event, stream) { + if (this.audioSource) { + this.audioSource.disconnect(); + } + // Connect to audioProcessor. + this.audioSource = context.createMediaStreamSource(stream); + //console.log("got source", this.audioSource); + this.audioSource.connect(this.audioProcessor); + this.audioProcessor.connect(context.destination); + }, this)); + } + } }; @@ -112,11 +132,30 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ if (!mediaConstraints) { mediaConstraints = currentcall.mediaConstraints; } + this.mediaConstraints = mediaConstraints; + + return this.doGetUserMediaWithConstraints(mediaConstraints); + + }; + + UserMedia.prototype.doGetUserMediaWithConstraints = function(mediaConstraints) { + + if (!mediaConstraints) { + mediaConstraints = this.mediaConstraints; + } + + var constraints = $.extend(true, {}, mediaConstraints); + if (this.audioMute) { + constraints.audio = false; + } + if (this.videoMute) { + constraints.video = false; + } try { console.log('Requesting access to local media with mediaConstraints:\n' + - ' \'' + JSON.stringify(mediaConstraints) + '\'', mediaConstraints); - getUserMedia(mediaConstraints, _.bind(this.onUserMediaSuccess, this), _.bind(this.onUserMediaError, this)); + ' \'' + JSON.stringify(constraints) + '\'', constraints); + getUserMedia(constraints, _.bind(this.onUserMediaSuccess, this), _.bind(this.onUserMediaError, this)); this.started = true; return true; } catch (e) { @@ -134,27 +173,7 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ return; } - // Get notified of end events. - stream.onended = _.bind(function(event) { - console.log("User media stream ended."); - if (this.started) { - this.stop(); - } - }, this); - - if (this.audioProcessor && context.createMediaStreamSource) { - // Connect to audioProcessor. - this.audioSource = context.createMediaStreamSource(stream); - //console.log("got source", this.audioSource); - this.audioSource.connect(this.audioProcessor); - this.audioProcessor.connect(context.destination); - } - this.localStream = stream; - - // Let webrtc handle the rest. - setTimeout(_.bind(function() { - this.e.triggerHandler("mediasuccess", [this]); - }, this), this.delay); + this.onLocalStream(stream); }; @@ -170,6 +189,36 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ }; + UserMedia.prototype.onLocalStream = function(stream) { + + var oldStream = this.localStream; + if (oldStream) { + oldStream.onended = function() {}; + oldStream.stop(); + setTimeout(_.bind(function() { + this.e.triggerHandler("mediachanged", [this]); + }, this), 0); + } else { + // Let webrtc handle the rest. + setTimeout(_.bind(function() { + this.e.triggerHandler("mediasuccess", [this]); + }, this), this.delay); + } + + // Get notified of end events. + stream.onended = _.bind(function(event) { + console.log("User media stream ended."); + if (this.started) { + this.stop(); + } + }, this); + + // Set new stream. + this.localStream = stream; + this.e.triggerHandler("localstream", [stream, oldStream, this]); + + }; + UserMedia.prototype.stop = function() { this.started = false; @@ -186,6 +235,9 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ this.audioProcessor.disconnect() } this.audioLevel = 0; + this.audioMute = false; + this.videoMute = false; + this.mediaConstraints = null; console.log("Stopped user media."); this.e.triggerHandler("stopped", [this]); @@ -198,6 +250,8 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ UserMedia.prototype.applyAudioMute = function(mute) { + this.audioMute = !!mute; + if (this.localStream) { var audioTracks = this.localStream.getAudioTracks(); @@ -224,9 +278,11 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ UserMedia.prototype.applyVideoMute = function(mute) { + var m = !!mute; + if (this.localStream) { - var videoTracks = this.localStream.getVideoTracks(); + /*var videoTracks = this.localStream.getVideoTracks(); if (videoTracks.length === 0) { //console.log('No local video available.'); return; @@ -240,8 +296,15 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ console.log("Local video muted.") } else { console.log("Local video unmuted.") + }*/ + + if (this.videoMute !== m) { + this.videoMute = m; + this.doGetUserMediaWithConstraints(); } + } else { + this.videoMute = m; } return mute; @@ -253,6 +316,14 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ console.log("Add usermedia stream to peer connection", pc, this.localStream); if (this.localStream) { pc.addStream(this.localStream); + this.e.on("localstream", _.bind(function(event, stream, oldstream) { + // Update stream support. + if (oldstream) { + pc.removeStream(oldstream); + pc.addStream(stream); + console.log("Updated usermedia stream at peer connection", pc, stream); + } + }, this)) } }; @@ -268,7 +339,6 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ UserMedia.prototype.attachMediaStream = function(video) { - //console.log("attach", video, this.localStream); attachMediaStream(video, this.localStream); }; diff --git a/static/js/mediastream/webrtc.js b/static/js/mediastream/webrtc.js index 6a3a2366..97c7f069 100644 --- a/static/js/mediastream/webrtc.js +++ b/static/js/mediastream/webrtc.js @@ -117,6 +117,10 @@ function($, _, PeerCall, PeerConference, PeerXfer, PeerScreenshare, UserMedia, u // Start always, no matter what. this.maybeStart(); }, this)); + this.usermedia.e.on("mediachanged", _.bind(function() { + // Propagate media change events. + this.e.triggerHandler("usermedia", [this.usermedia]); + }, this)); }; @@ -226,32 +230,36 @@ function($, _, PeerCall, PeerConference, PeerXfer, PeerScreenshare, UserMedia, u switch (type) { case "Offer": - var busy = false; - var conference = null; - if (this.currentcall.from !== from) { + console.log("Offer process."); + if (this.settings.stereo) { + data.sdp = utils.addStereo(data.sdp); + } + targetcall = this.findTargetCall(from); + if (targetcall) { + // Hey we know this call. + targetcall.setRemoteDescription(new RTCSessionDescription(data), _.bind(function(sessionDescription, currentcall) { + if (currentcall === this.currentcall) { + // Main call. + this.e.triggerHandler("peercall", [this.currentcall]); + } + currentcall.createAnswer(_.bind(function(sessionDescription, currentcall) { + console.log("Sending answer", sessionDescription, currentcall.id); + this.api.sendAnswer(currentcall.id, sessionDescription); + }, this)); + }, this)); + } else { + // No target call. Check conference auto answer support. if (this.currentconference && this.currentconference.id === data._conference) { console.log("Received conference Offer -> auto.", from, data._conference); - conference = data._conference; - // clean own internal data before feeding into browser. + // Clean own internal data before feeding into browser. delete data._conference; - } else { - console.log("Received Offer from unknown id -> busy.", from, this.currentconference); - busy = true; + this.currentconference.autoAnswer(from, new RTCSessionDescription(data)); + break; } - } - if (busy) { + // Cannot do anything with this offer, reply with busy. + console.log("Received Offer from unknown id -> busy.", from); this.api.sendBye(from, "busy"); this.e.triggerHandler("busy", [from, to2, to]); - return; - } - console.log("Offer process."); - if (this.settings.stereo) { - data.sdp = utils.addStereo(data.sdp); - } - if (conference) { - this.currentconference.autoAnswer(from, new RTCSessionDescription(data)); - } else { - this.currentcall.setRemoteDescription(new RTCSessionDescription(data), _.bind(this.doAnswer, this)); } break; case "Candidate": @@ -428,16 +436,6 @@ function($, _, PeerCall, PeerConference, PeerXfer, PeerScreenshare, UserMedia, u }; - WebRTC.prototype.doAnswer = function() { - - this.e.triggerHandler("peercall", [this.currentcall]); - this.currentcall.createAnswer(_.bind(function(sessionDescription, currentcall) { - console.log("Sending answer", sessionDescription, currentcall.id); - this.api.sendAnswer(currentcall.id, sessionDescription); - }, this)); - - }; - WebRTC.prototype.doXfer = function(id, token, options) { var registeredToken = tokens.get(token); diff --git a/static/js/services/videolayout.js b/static/js/services/videolayout.js index f53b5a1d..bf32084d 100644 --- a/static/js/services/videolayout.js +++ b/static/js/services/videolayout.js @@ -23,14 +23,14 @@ define(["jquery", "underscore", "modernizr", "injectCSS"], function($, _, Modern var dynamicCSSContainer = "audiovideo-dynamic"; var renderers = {}; - var getRemoteVideoSize = function(videos, peers) { + var getRemoteVideoSize = function(videos, streams) { var size = { width: 1920, height: 1080 } if (videos.length) { if (videos.length === 1) { - var remoteVideo = peers[videos[0]].element.find("video").get(0); + var remoteVideo = streams[videos[0]].element.find("video").get(0); if (remoteVideo) { size.width = remoteVideo.videoWidth; size.height = remoteVideo.videoHeight; @@ -51,7 +51,7 @@ define(["jquery", "underscore", "modernizr", "injectCSS"], function($, _, Modern OnePeople.prototype.name = "onepeople"; - OnePeople.prototype.render = function(container, size, scope, videos, peers) { + OnePeople.prototype.render = function(container, size, scope, videos, streams) { if (this.closed) { return; @@ -61,7 +61,7 @@ define(["jquery", "underscore", "modernizr", "injectCSS"], function($, _, Modern var videoHeight; if (videos.length) { - var remoteSize = getRemoteVideoSize(videos, peers); + var remoteSize = getRemoteVideoSize(videos, streams); videoWidth = remoteSize.width; videoHeight = remoteSize.height; } @@ -235,25 +235,25 @@ define(["jquery", "underscore", "modernizr", "injectCSS"], function($, _, Modern }; - ConferenceKiosk.prototype.render = function(container, size, scope, videos, peers) { + ConferenceKiosk.prototype.render = function(container, size, scope, videos, streams) { var big = this.big; if (big) { var currentbigpeerid = this.big.data("peerid"); - if (!peers[currentbigpeerid]) { + if (!streams[currentbigpeerid]) { console.log("Current big peer is no longer there", currentbigpeerid); this.big = big = null; } } if (!big) { if (videos.length) { - this.makeBig(peers[videos[0]].element); + this.makeBig(streams[videos[0]].element); this.bigVideo.style.opacity = 1; } } - var remoteSize = getRemoteVideoSize(videos, peers); + var remoteSize = getRemoteVideoSize(videos, streams); var aspectRatio = remoteSize.width / remoteSize.height; var innerHeight = size.height - 110; var innerWidth = size.width; @@ -304,18 +304,18 @@ define(["jquery", "underscore", "modernizr", "injectCSS"], function($, _, Modern Classroom.prototype = Object.create(ConferenceKiosk.prototype); Classroom.prototype.constructor = Classroom; Classroom.prototype.name = "classroom"; - Classroom.prototype.render = function(container, size, scope, videos, peers) { + Classroom.prototype.render = function(container, size, scope, videos, streams) { var big = this.big; if (big) { var currentbigpeerid = this.big.data("peerid"); - if (!peers[currentbigpeerid]) { + if (!streams[currentbigpeerid]) { console.log("Current big peer is no longer there", currentbigpeerid); this.big = big = null; } } if (!big) { if (videos.length) { - this.makeBig(peers[videos[0]].element); + this.makeBig(streams[videos[0]].element); this.bigVideo.style.opacity = 1; } @@ -345,8 +345,8 @@ define(["jquery", "underscore", "modernizr", "injectCSS"], function($, _, Modern return r; }; - var videos = _.keys(controller.peers); - var peers = controller.peers; + var videos = _.keys(controller.streams); + var streams = controller.streams; var container = scope.container; var layoutparent = scope.layoutparent; @@ -370,7 +370,7 @@ define(["jquery", "underscore", "modernizr", "injectCSS"], function($, _, Modern } } - return current.render(container, size, scope, videos, peers); + return current.render(container, size, scope, videos, streams); }, register: function(name, impl) { diff --git a/static/partials/audiovideopeer.html b/static/partials/audiovideopeer.html index 4ccc80a4..f4b0e1a4 100644 --- a/static/partials/audiovideopeer.html +++ b/static/partials/audiovideopeer.html @@ -1,4 +1,4 @@ -
+
{{peerid|displayName}}