From ccdd51fd7457811454bfb888842202bb9d0395c7 Mon Sep 17 00:00:00 2001 From: Simon Eisenmann Date: Mon, 6 Apr 2015 00:19:07 +0200 Subject: [PATCH 1/4] Worked to get screen sharing to FF. Use new mediaDevices service. Prepare to support video constraints in FF>=38. Disable HD constraints for Firefox. --- .../js/controllers/mediastreamcontroller.js | 2 +- static/js/directives/buddypicturecapture.js | 8 +- static/js/directives/settings.js | 58 +++++------ static/js/mediastream/usermedia.js | 98 ++++++++++++++++++- static/js/services/constraints.js | 18 +++- static/js/services/mediadevices.js | 62 ++++++++++++ static/js/services/screensharing.js | 23 ++++- static/js/services/services.js | 9 +- static/partials/settings.html | 28 +++--- 9 files changed, 249 insertions(+), 57 deletions(-) create mode 100644 static/js/services/mediadevices.js diff --git a/static/js/controllers/mediastreamcontroller.js b/static/js/controllers/mediastreamcontroller.js index 313ffadf..c8e931ee 100644 --- a/static/js/controllers/mediastreamcontroller.js +++ b/static/js/controllers/mediastreamcontroller.js @@ -111,7 +111,7 @@ define(['jquery', 'underscore', 'angular', 'bigscreen', 'moment', 'sjcl', 'moder // Add support status. $scope.supported = { screensharing: screensharing.supported, - renderToAssociatedSink: $window.navigator.platform.indexOf("Win") === 0 + constraints: constraints.supported } // Default scope data. diff --git a/static/js/directives/buddypicturecapture.js b/static/js/directives/buddypicturecapture.js index 7d09ccce..09a21951 100644 --- a/static/js/directives/buddypicturecapture.js +++ b/static/js/directives/buddypicturecapture.js @@ -25,7 +25,7 @@ define(['jquery', 'underscore', 'text!partials/buddypicturecapture.html'], funct // buddyPictureCapture return ["$compile", "$window", function($compile, $window) { - var controller = ['$scope', 'safeApply', '$timeout', '$q', function($scope, safeApply, $timeout, $q) { + var controller = ['$scope', 'safeApply', '$timeout', '$q', "mediaDevices", function($scope, safeApply, $timeout, $q, mediaDevices) { // Buddy picutre capture size. $scope.captureSize = { @@ -126,16 +126,16 @@ define(['jquery', 'underscore', 'text!partials/buddypicturecapture.html'], funct }] }; } - $window.getUserMedia({ + mediaDevices.getUserMedia({ video: videoConstraints - }, function(stream) { + }).then(function(stream) { $scope.showTakePicture = true; localStream = stream; $scope.waitingForPermission = false; $window.attachMediaStream($scope.video, stream); safeApply($scope); videoAllowed.resolve(true); - }, function(error) { + }).catch(function(error) { console.error('Failed to get access to local media. Error code was ' + error.code); $scope.waitingForPermission = false; safeApply($scope); diff --git a/static/js/directives/settings.js b/static/js/directives/settings.js index 26d2cb0e..925b03ad 100644 --- a/static/js/directives/settings.js +++ b/static/js/directives/settings.js @@ -171,29 +171,29 @@ define(['jquery', 'underscore', 'text!partials/settings.html'], function($, _, t $timeout($scope.maybeShowSettings); }); - constraints.e.on("refresh", function(event, constraints) { + constraints.e.on("refresh", function(event, c) { var settings = $scope.master.settings; // Assert that selected devices are there. (function() { - var deferred = constraints.defer(); + var deferred = c.defer(); mediaSources.refresh(function() { $scope.checkDefaultMediaSources(); // Select microphone device by id. if (settings.microphoneId) { - constraints.add("audio", "sourceId", settings.microphoneId); + c.add("audio", "sourceId", settings.microphoneId); } // Select camera by device id. if (settings.cameraId) { - constraints.add("video", "sourceId", settings.cameraId); + c.add("video", "sourceId", settings.cameraId); } if (!mediaSources.hasAudio()) { - constraints.disable('audio'); + c.disable('audio'); console.info("Disabled audio input as no audio source was found."); } if (!mediaSources.hasVideo()) { - constraints.disable('video'); + c.disable('video'); console.info("Disabled video input as no video source was found."); } deferred.resolve("complete"); @@ -201,7 +201,7 @@ define(['jquery', 'underscore', 'text!partials/settings.html'], function($, _, t })(); // Chrome only constraints. - if ($scope.isChrome) { + if (constraints.supported.chrome) { // Chrome specific constraints overview: // https://code.google.com/p/webrtc/source/browse/trunk/talk/app/webrtc/mediaconstraintsinterface.cc @@ -211,24 +211,24 @@ define(['jquery', 'underscore', 'text!partials/settings.html'], function($, _, t // Experimental audio settings. if (settings.experimental.enabled) { - constraints.add("audio", "googEchoCancellation", true); // defaults to true - constraints.add("audio", "googEchoCancellation2", settings.experimental.audioEchoCancellation2 && true); // defaults to false in Chrome - constraints.add("audio", "googAutoGainControl", true); // defaults to true - constraints.add("audio", "googAutoGainControl2", settings.experimental.audioAutoGainControl2 && true); // defaults to false in Chrome - constraints.add("audio", "googNoiseSuppression", true); // defaults to true - constraints.add("audio", "googgNoiseSuppression2", settings.experimental.audioNoiseSuppression2 && true); // defaults to false in Chrome - constraints.add("audio", "googHighpassFilter", true); // defaults to true - constraints.add("audio", "googTypingNoiseDetection", settings.experimental.audioTypingNoiseDetection && true); // defaults to true in Chrome + c.add("audio", "googEchoCancellation", true); // defaults to true + c.add("audio", "googEchoCancellation2", settings.experimental.audioEchoCancellation2 && true); // defaults to false in Chrome + c.add("audio", "googAutoGainControl", true); // defaults to true + c.add("audio", "googAutoGainControl2", settings.experimental.audioAutoGainControl2 && true); // defaults to false in Chrome + c.add("audio", "googNoiseSuppression", true); // defaults to true + c.add("audio", "googgNoiseSuppression2", settings.experimental.audioNoiseSuppression2 && true); // defaults to false in Chrome + c.add("audio", "googHighpassFilter", true); // defaults to true + c.add("audio", "googTypingNoiseDetection", settings.experimental.audioTypingNoiseDetection && true); // defaults to true in Chrome } - if ($scope.supported.renderToAssociatedSink) { + if (constraints.supported.renderToAssociatedSink) { // When true uses the default communications device on Windows. // https://codereview.chromium.org/155863003 - constraints.add("audio", "googDucking", true); // defaults to true on Windows. + c.add("audio", "googDucking", true); // defaults to true on Windows. // Chrome will start rendering mediastream output to an output device that's associated with // the input stream that was opened via getUserMedia. // https://chromiumcodereview.appspot.com/23558010 - constraints.add("audio", "chromeRenderToAssociatedSink", settings.audioRenderToAssociatedSkin && true); // defaults to false in Chrome + c.add("audio", "chromeRenderToAssociatedSink", settings.audioRenderToAssociatedSkin && true); // defaults to false in Chrome } // Experimental video settings. @@ -236,44 +236,44 @@ define(['jquery', 'underscore', 'text!partials/settings.html'], function($, _, t // Changes the way the video encoding adapts to the available bandwidth. // https://code.google.com/p/webrtc/issues/detail?id=3351 - constraints.add(["video", "screensharing"], "googLeakyBucket", settings.experimental.videoLeakyBucket && true); // defaults to false in Chrome + c.add(["video", "screensharing"], "googLeakyBucket", settings.experimental.videoLeakyBucket && true); // defaults to false in Chrome // Removes the noise in the captured video stream at the expense of CPU. - constraints.add(["video", "screensharing"], "googNoiseReduction", settings.experimental.videoNoiseReduction && true); // defaults to false in Chrome - constraints.add("pc", "googCpuOveruseDetection", settings.experimental.videoCpuOveruseDetection && true); // defaults to true in Chrome + c.add(["video", "screensharing"], "googNoiseReduction", settings.experimental.videoNoiseReduction && true); // defaults to false in Chrome + c.add("pc", "googCpuOveruseDetection", settings.experimental.videoCpuOveruseDetection && true); // defaults to true in Chrome } + } + + if (constraints.supported.audioVideo) { + // Set video quality. var videoQuality = videoQualityMap[settings.videoQuality]; if (videoQuality) { var mandatory = videoQuality.mandatory; _.forEach(videoQuality, function(v, k) { if (k !== "mandatory") { - constraints.add("video", k, v, mandatory ? false : true); + c.add("video", k, v, mandatory ? false : true); } }); if (mandatory) { _.forEach(mandatory, function(v, k) { - constraints.add("video", k, v, true); + c.add("video", k, v, true); }); } } // Set max frame rate if any was selected. if (settings.maxFrameRate && settings.maxFrameRate != "auto") { - constraints.add("video", "maxFrameRate", parseInt(settings.maxFrameRate, 10), true); + c.add("video", "maxFrameRate", parseInt(settings.maxFrameRate, 10), true); } // Disable AEC if stereo. // https://github.com/webrtc/apprtc/issues/23 if (settings.sendStereo) { - constraints.add("audio", "echoCancellation", false); + c.add("audio", "echoCancellation", false); } - } else { - - // Other browsers constraints (there are none as of now.); - } }); diff --git a/static/js/mediastream/usermedia.js b/static/js/mediastream/usermedia.js index ebc7dc17..0761fbce 100644 --- a/static/js/mediastream/usermedia.js +++ b/static/js/mediastream/usermedia.js @@ -30,6 +30,100 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ // Firefox and Chrome with Firefox interop. var enableRenegotiationSupport = false; + // Converter helpers to convert media constraints to new API. + var mergeConstraints = function(constraints, k, v, mandatory) { + var prefix = k.substring(0, 3); + switch (prefix) { + case "min": + case "max": + var suffix = k[3].toLowerCase()+k.substring(4); + if (!constraints.hasOwnProperty(suffix)) { + constraints[suffix]={}; + } + if (mandatory && prefix === "min" && constraints[suffix].hasOwnProperty(prefix)) { + // Use existing min constraint as ideal. + constraints[suffix].ideal = constraints[suffix].min; + } + constraints[suffix][prefix]=v; + break; + default: + constraints[k] = v; + break; + } + }; + var convertConstraints = function(constraints) { + if (!constraints) { + return false; + } + if (!constraints.hasOwnProperty("optional") && !constraints.hasOwnProperty("mandatory")) { + // No old style members. + return constraints; + } + var c = {}; + // Process optional constraints. + if (constraints.optional) { + _.each(constraints.optional, function(o) { + _.each(o, function(v, k) { + mergeConstraints(c, k, v); + }) + }); + } + // Process mandatory constraints. + if (constraints.mandatory) { + _.each(constraints.mandatory, function(v, k) { + mergeConstraints(c, k, v, true); + }); + } + // Fastpath. + if (_.isEmpty(c)) { + return true; + } + // Use ideal if there is only one value set. + _.each(c, function(v, k) { + if (_.isObject(v)) { + var values = _.values(v); + if (values.length === 1) { + // Use as ideal value if only one given. + c[k] = {ideal: values[0]}; + } + } + }); + if (window.webrtcDetectedBrowser === "firefox" && window.webrtcDetectedVersion < 38) { + // Firefox < 38 needs a extra require field. + var r = []; + if (c.height) { + r.push("height"); + } + if (c.width) { + r.push("width"); + } + if (r.length) { + c.require = r; + } + } + return c; + }; + // Adapter to support navigator.mediaDevices API. + // http://w3c.github.io/mediacapture-main/getusermedia.html#mediadevices + var getUserMedia = (function() { + if (window.navigator.mediaDevices) { + console.info("Enabled mediaDevices adapter ..."); + return function(constraints, success, error) { + // Full constraints syntax with plain values and ideal-algorithm supported in FF38+. + // Note on FF32-37: Plain values and ideal are not supported. + // See https://wiki.mozilla.org/Media/getUserMedia for details. + // Examples here: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia + var c = {audio: convertConstraints(constraints.audio), video: convertConstraints(constraints.video)}; + // mediaDevices API returns a promise. + console.log("Constraints for mediaDevices", c); + window.navigator.mediaDevices.getUserMedia(c).then(success).catch(error); + } + } else { + // Use existing adapter. + return window.getUserMedia; + } + })(); + var UserMedia = function(options) { this.options = $.extend({}, options); @@ -128,7 +222,7 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ error_cb.apply(this, args); }; try { - window.getUserMedia({ + getUserMedia({ video: true, audio: true }, success_helper, error_helper); @@ -173,7 +267,7 @@ define(['jquery', 'underscore', 'audiocontext', 'webrtc.adapter'], function($, _ try { console.log('Requesting access to local media with mediaConstraints:\n' + ' \'' + JSON.stringify(constraints) + '\'', constraints); - window.getUserMedia(constraints, _.bind(this.onUserMediaSuccess, this), _.bind(this.onUserMediaError, this)); + getUserMedia(constraints, _.bind(this.onUserMediaSuccess, this), _.bind(this.onUserMediaError, this)); this.started = true; return true; } catch (e) { diff --git a/static/js/services/constraints.js b/static/js/services/constraints.js index 09d5009d..6394a886 100644 --- a/static/js/services/constraints.js +++ b/static/js/services/constraints.js @@ -187,7 +187,23 @@ }, stun: function(stunData) { service.stun = stunData; - } + }, + supported: (function() { + var isChrome = $window.webrtcDetectedBrowser === "chrome"; + var isFirefox = $window.webrtcDetectedBrowser === "firefox"; + var version = $window.webrtcDetectedVersion; + // Constraints support table. + return { + // Chrome supports it. See https://wiki.mozilla.org/Media/getUserMedia for FF details. + audioVideo: isChrome || (isFirefox && version >= 38), + // Disable FF HD constraints for now (see https://bugzilla.mozilla.org/show_bug.cgi?id=1150539) + hdVideo: isChrome, + // Chrome supports this on Windows only. + renderToAssociatedSink: isChrome && $window.navigator.platform.indexOf("Win") === 0, + chrome: isChrome, + firefox: isFirefox + }; + })() }; }]; diff --git a/static/js/services/mediadevices.js b/static/js/services/mediadevices.js new file mode 100644 index 00000000..5680aa1a --- /dev/null +++ b/static/js/services/mediadevices.js @@ -0,0 +1,62 @@ +/* + * Spreed WebRTC. + * Copyright (C) 2013-2014 struktur AG + * + * This file is part of Spreed WebRTC. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +/* global Promise */ +"use strict"; +define(['webrtc.adapter'], function() { + + // mediaDevices + return ["$window", function($window) { + + var mediaDevices = $window.navigator.mediaDevices || {}; + var getUserMedia = (function() { + // Implement a Promise based wrapper around getUserMedia. + if (mediaDevices.getUserMedia) { + // mediaDevices calls return Promise native. + return mediaDevices.getUserMedia.bind(mediaDevices); + } else { + return function getUserMedia(constraints) { + return new Promise(function(resolve, reject) { + var onSuccess = function(stream) { + resolve(stream) + }; + var onError = function(error) { + reject(error); + }; + try { + $window.getUserMedia(constraints, onSuccess, onError); + } catch(err) { + onError(err); + } + }); + } + } + })(); + + // Public api. + return { + shim: mediaDevices.getUserMedia ? false : true, + getUserMedia: getUserMedia + } + + }]; + +}); \ No newline at end of file diff --git a/static/js/services/screensharing.js b/static/js/services/screensharing.js index 515c5a43..722b7bc4 100644 --- a/static/js/services/screensharing.js +++ b/static/js/services/screensharing.js @@ -121,10 +121,27 @@ define(['underscore', 'webrtc.adapter'], function(_) { } - } else { - // Currently Chrome only - sorry. - // Firefox 33 might get screen sharing support. + } else if ($window.webrtcDetectedBrowser === "firefox") { + + // Firefox 36 got screen sharing support. // See https://bugzilla.mozilla.org/show_bug.cgi?id=923225 + if ($window.webrtcDetectedVersion >= 36) { + this.supported = true; + this.prepare = function(options) { + // To work, the current domain must be whitelisted in + // media.getusermedia.screensharing.allowed_domains (about:config). + // See https://wiki.mozilla.org/Screensharing for reference. + var d = $q.defer() + var opts = _.extend({ + mediaSource: "screen" + }, options); + d.resolve(opts); + return d.promise; + }; + } + + } else { + // No support for screen sharing. } // Auto install support. diff --git a/static/js/services/services.js b/static/js/services/services.js index 18c5eeec..b0fb7d02 100644 --- a/static/js/services/services.js +++ b/static/js/services/services.js @@ -66,7 +66,8 @@ define([ 'services/resturl', 'services/roompin', 'services/constraints', - 'services/modules'], function(_, + 'services/modules', + 'services/mediadevices'], function(_, desktopNotify, playSound, safeApply, @@ -110,7 +111,8 @@ rooms, restURL, roompin, constraints, -modules) { +modules, +mediaDevices) { var services = { desktopNotify: desktopNotify, @@ -156,7 +158,8 @@ modules) { restURL: restURL, roompin: roompin, constraints: constraints, - modules: modules + modules: modules, + mediaDevices: mediaDevices }; var initialize = function(angModule) { diff --git a/static/partials/settings.html b/static/partials/settings.html index 341ee594..c894303e 100644 --- a/static/partials/settings.html +++ b/static/partials/settings.html @@ -66,7 +66,7 @@ - + {{_('Media')}}
@@ -82,14 +82,14 @@
-
+
- - + +
@@ -130,7 +130,7 @@
-
+
@@ -143,7 +143,7 @@
-
+
@@ -154,7 +154,7 @@
-
+
@@ -165,7 +165,7 @@
-
+
@@ -178,7 +178,7 @@
-
+
@@ -191,7 +191,7 @@
-
+
@@ -203,7 +203,7 @@
-
+
@@ -217,7 +217,7 @@
-
+
@@ -228,7 +228,7 @@
-
+
@@ -241,7 +241,7 @@
-
+
From 0c6d47b38cfee5bf85cb4aefbf33075478949bba Mon Sep 17 00:00:00 2001 From: Simon Eisenmann Date: Tue, 7 Apr 2015 19:23:54 +0200 Subject: [PATCH 2/4] Enable HD constraints for FF >= 38 as https://bugzilla.mozilla.org/show_bug.cgi?id=1151628 was fixed. Lets hope it will land in 38. --- static/js/services/constraints.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/js/services/constraints.js b/static/js/services/constraints.js index 6394a886..99d94bc0 100644 --- a/static/js/services/constraints.js +++ b/static/js/services/constraints.js @@ -196,8 +196,8 @@ return { // Chrome supports it. See https://wiki.mozilla.org/Media/getUserMedia for FF details. audioVideo: isChrome || (isFirefox && version >= 38), - // Disable FF HD constraints for now (see https://bugzilla.mozilla.org/show_bug.cgi?id=1150539) - hdVideo: isChrome, + // HD constraints in Chrome no issue. In FF we just assume MJPEG is fixed with FF 38 (see https://bugzilla.mozilla.org/show_bug.cgi?id=1151628). + hdVideo: isChrome || (isFirefox && version >= 38), // Chrome supports this on Windows only. renderToAssociatedSink: isChrome && $window.navigator.platform.indexOf("Win") === 0, chrome: isChrome, From 5bea610568f3b0de046d80d00fa0e3387a7c9a72 Mon Sep 17 00:00:00 2001 From: Simon Eisenmann Date: Mon, 13 Apr 2015 18:32:27 +0200 Subject: [PATCH 3/4] Bumped constraints support for HD to FF 40 to make sure MJPEG works fine. --- static/js/services/constraints.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/static/js/services/constraints.js b/static/js/services/constraints.js index 99d94bc0..99224615 100644 --- a/static/js/services/constraints.js +++ b/static/js/services/constraints.js @@ -194,10 +194,10 @@ var version = $window.webrtcDetectedVersion; // Constraints support table. return { - // Chrome supports it. See https://wiki.mozilla.org/Media/getUserMedia for FF details. + // Chrome supports it. FF supports new spec starting 38. See https://wiki.mozilla.org/Media/getUserMedia for FF details. audioVideo: isChrome || (isFirefox && version >= 38), - // HD constraints in Chrome no issue. In FF we just assume MJPEG is fixed with FF 38 (see https://bugzilla.mozilla.org/show_bug.cgi?id=1151628). - hdVideo: isChrome || (isFirefox && version >= 38), + // HD constraints in Chrome no issue. In FF we MJPEG is fixed with 40 (see https://bugzilla.mozilla.org/show_bug.cgi?id=1151628). + hdVideo: isChrome || (isFirefox && version >= 40), // Chrome supports this on Windows only. renderToAssociatedSink: isChrome && $window.navigator.platform.indexOf("Win") === 0, chrome: isChrome, From c5a91562fcdc1bf44e1d7862724364e75020a7b3 Mon Sep 17 00:00:00 2001 From: Simon Eisenmann Date: Tue, 14 Apr 2015 12:50:27 +0200 Subject: [PATCH 4/4] Added selection to make screen, window and application possible in Firefox. --- static/js/services/screensharing.js | 35 +++++++++++++++++++----- static/partials/screensharedialogff.html | 22 +++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 static/partials/screensharedialogff.html diff --git a/static/js/services/screensharing.js b/static/js/services/screensharing.js index 722b7bc4..5a5f572b 100644 --- a/static/js/services/screensharing.js +++ b/static/js/services/screensharing.js @@ -20,10 +20,22 @@ */ "use strict"; -define(['underscore', 'webrtc.adapter'], function(_) { +define(['underscore', 'text!partials/screensharedialogff.html', 'webrtc.adapter'], function(_, screenshareDialogFF) { + + var screenshareDialogFFController = ["$scope", "$modalInstance", "data", function($scope, $modalInstance, data) { + $scope.data = data; + $scope.cancel = function() { + $modalInstance.close(null); + }; + $scope.ok = function() { + $modalInstance.close($scope.data.selection); + }; + }]; // screensharing - return ["$window", "$q", "$timeout", "chromeExtension", function($window, $q, $timeout, chromeExtension) { + return ["$window", "$q", "$timeout", "chromeExtension", "dialogs", "$templateCache", function($window, $q, $timeout, chromeExtension, dialogs, $templateCache) { + + $templateCache.put('/dialogs/screensharedialogff.html', screenshareDialogFF); var Screensharing = function() { this.autoinstall = false; @@ -131,11 +143,20 @@ define(['underscore', 'webrtc.adapter'], function(_) { // To work, the current domain must be whitelisted in // media.getusermedia.screensharing.allowed_domains (about:config). // See https://wiki.mozilla.org/Screensharing for reference. - var d = $q.defer() - var opts = _.extend({ - mediaSource: "screen" - }, options); - d.resolve(opts); + var d = $q.defer(); + var dlg = dialogs.create('/dialogs/screensharedialogff.html', screenshareDialogFFController, {selection: "screen"}, {}); + dlg.result.then(function(source) { + if (source) { + var opts = _.extend({ + mediaSource: source + }, options); + d.resolve(opts); + } else { + d.resolve(null); + } + }, function(err) { + d.resolve(null); + }); return d.promise; }; } diff --git a/static/partials/screensharedialogff.html b/static/partials/screensharedialogff.html new file mode 100644 index 00000000..86a56687 --- /dev/null +++ b/static/partials/screensharedialogff.html @@ -0,0 +1,22 @@ +
+ + + +
\ No newline at end of file