webrtc: move codec and bitrate settings on client side (#1990)

internal/core/webrtc_http_server.go

@@ -301,11 +301,6 @@ func (s *webRTCHTTPServer) onRequest(ctx *gin.Context) {
 				remoteAddr: ctx.ClientIP(),
 				offer:      offer,
 				publish:    (fname == "whip"),
-				videoCodec:   ctx.Query("video_codec"),
-				audioCodec:   ctx.Query("audio_codec"),
-				videoBitrate: ctx.Query("video_bitrate"),
-				audioBitrate: ctx.Query("audio_bitrate"),
-				audioVoice:   ctx.Query("audio_voice") == "true",
 			})
 			if res.err != nil {
 				if res.errStatusCode != 0 {

internal/core/webrtc_manager.go

@@ -134,11 +134,6 @@ type webRTCSessionNewReq struct {
 	remoteAddr string
 	offer      []byte
 	publish    bool
-	videoCodec   string
-	audioCodec   string
-	videoBitrate string
-	audioBitrate string
-	audioVoice   bool
 	res        chan webRTCSessionNewRes
 }
 

internal/core/webrtc_pc.go

@@ -11,22 +11,21 @@ import (
 	"github.com/bluenviron/mediamtx/internal/logger"
 )
 
-var videoCodecs = map[string][]webrtc.RTPCodecParameters{
+var videoCodecs = []webrtc.RTPCodecParameters{
-	"av1": {{
+	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:  webrtc.MimeTypeAV1,
 			ClockRate: 90000,
 		},
 		PayloadType: 96,
-	}},
+	},
-	"vp9": {
 	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:    webrtc.MimeTypeVP9,
 			ClockRate:   90000,
 			SDPFmtpLine: "profile-id=0",
 		},
-			PayloadType: 96,
+		PayloadType: 97,
 	},
 	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
@@ -34,28 +33,35 @@ var videoCodecs = map[string][]webrtc.RTPCodecParameters{
 			ClockRate:   90000,
 			SDPFmtpLine: "profile-id=1",
 		},
-			PayloadType: 96,
+		PayloadType: 98,
-		},
 	},
-	"vp8": {{
+	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:  webrtc.MimeTypeVP8,
 			ClockRate: 90000,
 		},
-		PayloadType: 96,
+		PayloadType: 99,
-	}},
+	},
-	"h264": {{
+	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:    webrtc.MimeTypeH264,
 			ClockRate:   90000,
 			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
 		},
-		PayloadType: 96,
+		PayloadType: 100,
-	}},
+	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    webrtc.MimeTypeH264,
+			ClockRate:   90000,
+			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+		},
+		PayloadType: 101,
+	},
 }
 
-var audioCodecs = map[string][]webrtc.RTPCodecParameters{
+var audioCodecs = []webrtc.RTPCodecParameters{
-	"opus": {{
+	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:    webrtc.MimeTypeOpus,
 			ClockRate:   48000,
@@ -63,28 +69,28 @@ var audioCodecs = map[string][]webrtc.RTPCodecParameters{
 			SDPFmtpLine: "minptime=10;useinbandfec=1",
 		},
 		PayloadType: 111,
-	}},
+	},
-	"g722": {{
+	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:  webrtc.MimeTypeG722,
 			ClockRate: 8000,
 		},
 		PayloadType: 9,
-	}},
+	},
-	"pcmu": {{
+	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:  webrtc.MimeTypePCMU,
 			ClockRate: 8000,
 		},
 		PayloadType: 0,
-	}},
+	},
-	"pcma": {{
+	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:  webrtc.MimeTypePCMA,
 			ClockRate: 8000,
 		},
 		PayloadType: 8,
-	}},
+	},
 }
 
 type peerConnection struct {
@@ -98,8 +104,6 @@ type peerConnection struct {
 }
 
 func newPeerConnection(
-	videoCodec string,
-	audioCodec string,
 	iceServers []webrtc.ICEServer,
 	iceHostNAT1To1IPs []string,
 	iceUDPMux ice.UDPMux,
@@ -124,45 +128,19 @@ func newPeerConnection(
 
 	mediaEngine := &webrtc.MediaEngine{}
 
-	if videoCodec != "" || audioCodec != "" {
-		codec, ok := videoCodecs[videoCodec]
-		if ok {
-			for _, params := range codec {
-				err := mediaEngine.RegisterCodec(params, webrtc.RTPCodecTypeVideo)
-				if err != nil {
-					return nil, err
-				}
-			}
-		}
-
-		codec, ok = audioCodecs[audioCodec]
-		if ok {
-			for _, params := range codec {
-				err := mediaEngine.RegisterCodec(params, webrtc.RTPCodecTypeAudio)
-				if err != nil {
-					return nil, err
-				}
-			}
-		}
-	} else { // register all codecs
 	for _, codec := range videoCodecs {
-			for _, params := range codec {
+		err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo)
-				err := mediaEngine.RegisterCodec(params, webrtc.RTPCodecTypeVideo)
 		if err != nil {
 			return nil, err
 		}
 	}
-		}
 
 	for _, codec := range audioCodecs {
-			for _, params := range codec {
+		err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeAudio)
-				err := mediaEngine.RegisterCodec(params, webrtc.RTPCodecTypeAudio)
 		if err != nil {
 			return nil, err
 		}
 	}
-		}
-	}
 
 	interceptorRegistry := &interceptor.Registry{}
 	if err := webrtc.RegisterDefaultInterceptors(mediaEngine, interceptorRegistry); err != nil {

internal/core/webrtc_publish_index.html

@@ -183,7 +183,100 @@ const generateSdpFragment = (offerData, candidates) => {
     }
 
     return frag;
-}
+};
+
+const setCodec = (section, codec) => {
+    const lines = section.split('\r\n');
+    const lines2 = [];
+    const payloadFormats = [];
+
+    for (const line of lines) {
+        if (!line.startsWith('a=rtpmap:')) {
+            lines2.push(line);
+        } else {
+            if (line.toLowerCase().includes(codec)) {
+                payloadFormats.push(line.slice('a=rtpmap:'.length).split(' ')[0]);
+                lines2.push(line);
+            }
+        }
+    }
+
+    const lines3 = [];
+
+    for (const line of lines2) {
+        if (line.startsWith('a=fmtp:')) {
+            if (payloadFormats.includes(line.slice('a=fmtp:'.length).split(' ')[0])) {
+                lines3.push(line);
+            }
+        } else if (line.startsWith('a=rtcp-fb:')) {
+            if (payloadFormats.includes(line.slice('a=rtcp-fb:'.length).split(' ')[0])) {
+                lines3.push(line);
+            }
+        } else {
+            lines3.push(line);
+        }
+    }
+
+    return lines3.join('\r\n');
+};
+
+const setVideoBitrate = (section, bitrate) => {
+    let lines = section.split('\r\n');
+
+    for (let i = 0; i < lines.length; i++) {
+        if (lines[i].startsWith('c=')) {
+            lines = [...lines.slice(0, i+1), 'b=TIAS:' + (parseInt(bitrate) * 1024).toString(), ...lines.slice(i+1)];
+            break
+        }
+    }
+
+    return lines.join('\r\n');
+};
+
+const setAudioBitrate = (section, bitrate, voice) => {
+    let opusPayloadFormat = '';
+    let lines = section.split('\r\n');
+
+    for (let i = 0; i < lines.length; i++) {
+        if (lines[i].startsWith('a=rtpmap:') && lines[i].toLowerCase().includes('opus/')) {
+            opusPayloadFormat = lines[i].slice('a=rtpmap:'.length).split(' ')[0];
+            break;
+        }
+    }
+
+    if (opusPayloadFormat === '') {
+        return section;
+    }
+
+    for (let i = 0; i < lines.length; i++) {
+        if (lines[i].startsWith('a=fmtp:' + opusPayloadFormat + ' ')) {
+            if (voice) {
+                lines[i] = 'a=fmtp:' + opusPayloadFormat + ' minptime=10;useinbandfec=1;maxaveragebitrate='
+                    + (parseInt(bitrate) * 1024).toString();
+            } else {
+                lines[i] = 'a=fmtp:' + opusPayloadFormat + ' maxplaybackrate=48000;stereo=1;sprop-stereo=1;maxaveragebitrate'
+                    + (parseInt(bitrate) * 1024).toString();
+            }
+        }
+    }
+
+    return lines.join('\r\n');
+};
+
+const editAnswer = (answer, videoCodec, audioCodec, videoBitrate, audioBitrate, audioVoice) => {
+    const sections = answer.split('m=');
+
+    for (let i = 0; i < sections.length; i++) {
+        const section = sections[i];
+        if (section.startsWith('video')) {
+            sections[i] = setVideoBitrate(setCodec(section, videoCodec), videoBitrate);
+        } else if (section.startsWith('audio')) {
+            sections[i] = setAudioBitrate(setCodec(section, audioCodec), audioBitrate, audioVoice);
+        }
+    }
+
+    return sections.join('m=');
+};
 
 class Transmitter {
     constructor(stream) {
@@ -221,31 +314,21 @@ class Transmitter {
         });
 
         this.pc.createOffer()
-            .then((desc) => {
+            .then((offer) => this.onLocalOffer(offer));
-                this.offerData = parseOffer(desc.sdp);
+    }
-                this.pc.setLocalDescription(desc);
-
-                console.log("sending offer");
 
-                const videoCodec = document.getElementById('video_codec').value;
+    onLocalOffer(offer) {
-                const audioCodec = document.getElementById('audio_codec').value;
+        this.offerData = parseOffer(offer.sdp);
-                const videoBitrate = document.getElementById('video_bitrate').value;
+        this.pc.setLocalDescription(offer);
-                const audioBitrate = document.getElementById('audio_bitrate').value;
-                const audioVoice = document.getElementById('audio_voice').checked;
 
-                const p = new URLSearchParams(window.location.search);
+        console.log("sending offer");
-                p.set('video_codec', videoCodec);
-                p.set('audio_codec', audioCodec);
-                p.set('video_bitrate', videoBitrate);
-                p.set('audio_bitrate', audioBitrate);
-                p.set('audio_voice', audioVoice ? 'true' : 'false');
 
-                fetch(new URL('whip', window.location.href) + '?' + p.toString(), {
+        fetch(new URL('whip', window.location.href) + window.location.search, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/sdp',
             },
-                    body: desc.sdp,
+            body: offer.sdp,
         })
             .then((res) => {
                 if (res.status !== 201) {
@@ -254,7 +337,7 @@ class Transmitter {
                 this.eTag = res.headers.get('E-Tag');
                 return res.text();
             })
-                    .then((sdp) => this.onRemoteDescription(new RTCSessionDescription({
+            .then((sdp) => this.onRemoteAnswer(new RTCSessionDescription({
                 type: 'answer',
                 sdp,
             })))
@@ -262,7 +345,6 @@ class Transmitter {
                 console.log('error: ' + err);
                 this.scheduleRestart();
             });
-            });
     }
 
     onConnectionState() {
@@ -278,11 +360,23 @@ class Transmitter {
         }
     }
 
-    onRemoteDescription(answer) {
+    onRemoteAnswer(answer) {
 		if (this.restartTimeout !== null) {
 			return;
 		}
 
+        answer = new RTCSessionDescription({
+            type: 'answer',
+            sdp: editAnswer(
+                answer.sdp,
+                document.getElementById('video_codec').value,
+                document.getElementById('audio_codec').value,
+                document.getElementById('video_bitrate').value,
+                document.getElementById('audio_bitrate').value,
+                document.getElementById('audio_voice').value,
+            ),
+        });
+
         this.pc.setRemoteDescription(new RTCSessionDescription(answer));
 
         if (this.queuedCandidates.length !== 0) {
@@ -445,7 +539,7 @@ const populateCodecs = () => {
             for (const codec of ['av1/90000', 'vp9/90000', 'vp8/90000', 'h264/90000']) {
                 if (sdp.includes(codec)) {
                     const opt = document.createElement('option');
-                    opt.value = codec.split('/')[0];
+                    opt.value = codec;
                     opt.text = codec.split('/')[0].toUpperCase();
                     document.getElementById('video_codec').appendChild(opt);
                 }
@@ -454,7 +548,7 @@ const populateCodecs = () => {
             for (const codec of ['opus/48000', 'g722/8000', 'pcmu/8000', 'pcma/8000']) {
                 if (sdp.includes(codec)) {
                     const opt = document.createElement('option');
-                    opt.value = codec.split('/')[0];
+                    opt.value = codec;
                     opt.text = codec.split('/')[0].toUpperCase();
                     document.getElementById('audio_codec').appendChild(opt);
                 }

internal/core/webrtc_read_index.html

@@ -132,9 +132,12 @@ class WHEPClient {
         };
 
         this.pc.createOffer()
-            .then((desc) => {
+            .then((offer) => this.onLocalOffer(offer));
-                this.offerData = parseOffer(desc.sdp);
+    }
-                this.pc.setLocalDescription(desc);
+
+    onLocalOffer(offer) {
+        this.offerData = parseOffer(offer.sdp);
+        this.pc.setLocalDescription(offer);
 
         console.log("sending offer");
 
@@ -143,7 +146,7 @@ class WHEPClient {
             headers: {
                 'Content-Type': 'application/sdp',
             },
-                    body: desc.sdp,
+            body: offer.sdp,
         })
             .then((res) => {
                 if (res.status !== 201) {
@@ -152,7 +155,7 @@ class WHEPClient {
                 this.eTag = res.headers.get('E-Tag');
                 return res.text();
             })
-                    .then((sdp) => this.onRemoteDescription(new RTCSessionDescription({
+            .then((sdp) => this.onRemoteAnswer(new RTCSessionDescription({
                 type: 'answer',
                 sdp,
             })))
@@ -160,7 +163,6 @@ class WHEPClient {
                 console.log('error: ' + err);
                 this.scheduleRestart();
             });
-            });
     }
 
     onConnectionState() {
@@ -176,7 +178,7 @@ class WHEPClient {
         }
     }
 
-	onRemoteDescription(answer) {
+	onRemoteAnswer(answer) {
         if (this.restartTimeout !== null) {
             return;
         }

internal/core/webrtc_session.go

@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"net/http"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -14,7 +13,6 @@ import (
 	"github.com/bluenviron/gortsplib/v3/pkg/ringbuffer"
 	"github.com/google/uuid"
 	"github.com/pion/ice/v2"
-	"github.com/pion/sdp/v3"
 	"github.com/pion/webrtc/v3"
 
 	"github.com/bluenviron/mediamtx/internal/logger"
@@ -48,91 +46,6 @@ func mediasOfIncomingTracks(tracks []*webRTCIncomingTrack) media.Medias {
 	return ret
 }
 
-func findOpusPayloadFormat(attributes []sdp.Attribute) int {
-	for _, attr := range attributes {
-		if attr.Key == "rtpmap" && strings.Contains(attr.Value, "opus/") {
-			parts := strings.SplitN(attr.Value, " ", 2)
-			pl, err := strconv.ParseUint(parts[0], 10, 31)
-			if err == nil {
-				return int(pl)
-			}
-		}
-	}
-	return 0
-}
-
-func editAnswer(
-	offer *webrtc.SessionDescription,
-	videoBitrateStr string,
-	audioBitrateStr string,
-	audioVoice bool,
-) error {
-	var sd sdp.SessionDescription
-	err := sd.Unmarshal([]byte(offer.SDP))
-	if err != nil {
-		return err
-	}
-
-	if videoBitrateStr != "" {
-		videoBitrate, err := strconv.ParseUint(videoBitrateStr, 10, 31)
-		if err != nil {
-			return err
-		}
-
-		for _, media := range sd.MediaDescriptions {
-			if media.MediaName.Media == "video" {
-				media.Bandwidth = []sdp.Bandwidth{{
-					Type:      "TIAS",
-					Bandwidth: videoBitrate * 1024,
-				}}
-				break
-			}
-		}
-	}
-
-	if audioBitrateStr != "" {
-		audioBitrate, err := strconv.ParseUint(audioBitrateStr, 10, 31)
-		if err != nil {
-			return err
-		}
-
-		for _, media := range sd.MediaDescriptions {
-			if media.MediaName.Media == "audio" {
-				pl := findOpusPayloadFormat(media.Attributes)
-				if pl != 0 {
-					for i, attr := range media.Attributes {
-						if attr.Key == "fmtp" && strings.HasPrefix(attr.Value, strconv.FormatInt(int64(pl), 10)+" ") {
-							if audioVoice {
-								media.Attributes[i] = sdp.Attribute{
-									Key: "fmtp",
-									Value: strconv.FormatInt(int64(pl), 10) + " minptime=10;useinbandfec=1;maxaveragebitrate=" +
-										strconv.FormatUint(audioBitrate*1024, 10),
-								}
-							} else {
-								media.Attributes[i] = sdp.Attribute{
-									Key: "fmtp",
-									Value: strconv.FormatInt(int64(pl), 10) + " stereo=1;sprop-stereo=1;maxaveragebitrate=" +
-										strconv.FormatUint(audioBitrate*1024, 10),
-								}
-							}
-						}
-					}
-				}
-
-				break
-			}
-		}
-	}
-
-	enc, err := sd.Marshal()
-	if err != nil {
-		return err
-	}
-
-	offer.SDP = string(enc)
-	return nil
-}
-
 func gatherOutgoingTracks(medias media.Medias) ([]*webRTCOutgoingTrack, error) {
 	var tracks []*webRTCOutgoingTrack
 
@@ -322,8 +235,6 @@ func (s *webRTCSession) runPublish() (int, error) {
 	defer res.path.publisherRemove(pathPublisherRemoveReq{author: s})
 
 	pc, err := newPeerConnection(
-		s.req.videoCodec,
-		s.req.audioCodec,
 		s.parent.genICEServers(),
 		s.iceHostNAT1To1IPs,
 		s.iceUDPMux,
@@ -381,11 +292,6 @@ func (s *webRTCSession) runPublish() (int, error) {
 	tmp := pc.LocalDescription()
 	answer = *tmp
 
-	err = editAnswer(&answer, s.req.videoBitrate, s.req.audioBitrate, s.req.audioVoice)
-	if err != nil {
-		return http.StatusBadRequest, err
-	}
-
 	err = s.writeAnswer(&answer)
 	if err != nil {
 		return http.StatusBadRequest, err
@@ -451,8 +357,6 @@ func (s *webRTCSession) runRead() (int, error) {
 	}
 
 	pc, err := newPeerConnection(
-		"",
-		"",
 		s.parent.genICEServers(),
 		s.iceHostNAT1To1IPs,
 		s.iceUDPMux,