hls muxer: support reading Opus tracks (#1338)

README.md

@@ -22,7 +22,7 @@ And can be read from the server with:
 |--------|--------|------|
 |RTSP|UDP, UDP-Multicast, TCP, RTSPS|H264, H265, VP8, VP9, AV1, MPEG2, M-JPEG, MP3, MPEG4 Audio (AAC), Opus, G711, G722, LPCM and any RTP-compatible codec|
 |RTMP|RTMP, RTMPS|H264, MPEG4 Audio (AAC)|
-|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, H265, MPEG4 Audio (AAC)|
+|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, H265, MPEG4 Audio (AAC), Opus|
 |WebRTC||H264, VP8, VP9, Opus, G711, G722|
 
 Features:
@@ -908,7 +908,7 @@ where `mystream` is the name of a stream that is being published.
 
 Although the server can produce HLS with a variety of video and audio codecs (that are listed at the beginning of the README), not all browsers can read all codecs. You can check what codecs your browser can read by visiting this page:
 
-https://jsfiddle.net/7nwxmLto
+https://jsfiddle.net/4msrhudv
 
 If you want to increase the compatibility of the stream in order to support most browsers, you have to re-encode it by using the H264 and AAC codecs, for instance by using _FFmpeg_:
 
@@ -1067,9 +1067,9 @@ Related projects
 
 Standards
 
-* RTSP 1.0 https://datatracker.ietf.org/doc/html/rfc2326
-* RTSP 2.0 https://datatracker.ietf.org/doc/html/rfc7826
+* RTSP/RTP standards https://github.com/aler9/gortsplib#links
 * HTTP 1.1 https://datatracker.ietf.org/doc/html/rfc2616
 * HLS https://datatracker.ietf.org/doc/html/rfc8216
 * HLS v2 https://datatracker.ietf.org/doc/html/draft-pantos-hls-rfc8216bis
+* Opus in MP4/ISOBMFF https://opus-codec.org/docs/opus_in_isobmff.html
 * Golang project layout https://github.com/golang-standards/project-layout

go.mod

@@ -5,7 +5,7 @@ go 1.18
 require (
 	code.cloudfoundry.org/bytefmt v0.0.0
 	github.com/abema/go-mp4 v0.0.0
-	github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823
+	github.com/aler9/gortsplib/v2 v2.0.0-20230103153002-0ce435414414
 	github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gin-gonic/gin v1.8.1

go.sum

@@ -4,8 +4,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2c
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/aler9/go-mp4 v0.0.0-20221229200349-f3d01e787968 h1:wU8pLx4dc8bLB+JuVPWuGp+BoMkOabj98a0RmO3gqvw=
 github.com/aler9/go-mp4 v0.0.0-20221229200349-f3d01e787968/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
-github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823 h1:EFq9LqgA15drNgXj3hNlmAouxjMYb9jyyBq6hmjDO8U=
-github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
+github.com/aler9/gortsplib/v2 v2.0.0-20230103153002-0ce435414414 h1:pVyJ7Uuk5kdU/RhCepxJQJEC9hsrFgxIIw1mIHn02Zs=
+github.com/aler9/gortsplib/v2 v2.0.0-20230103153002-0ce435414414/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
 github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=
 github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U=
 github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=

internal/core/formatprocessor.go

@@ -25,6 +25,9 @@ func newFormatProcessor(forma format.Format, generateRTPPackets bool) (formatPro
 	case *format.MPEG4Audio:
 		return newFormatProcessorMPEG4Audio(forma, generateRTPPackets)
 
+	case *format.Opus:
+		return newFormatProcessorOpus(forma, generateRTPPackets)
+
 	default:
 		return newFormatProcessorGeneric(forma, generateRTPPackets)
 	}

internal/core/formatprocessor_opus.go

@@ -0,0 +1,89 @@
+package core
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/aler9/gortsplib/v2/pkg/format"
+	"github.com/aler9/gortsplib/v2/pkg/formatdecenc/rtpsimpleaudio"
+	"github.com/pion/rtp"
+)
+
+type dataOpus struct {
+	rtpPackets []*rtp.Packet
+	ntp        time.Time
+	pts        time.Duration
+	frame      []byte
+}
+
+func (d *dataOpus) getRTPPackets() []*rtp.Packet {
+	return d.rtpPackets
+}
+
+func (d *dataOpus) getNTP() time.Time {
+	return d.ntp
+}
+
+type formatProcessorOpus struct {
+	format  *format.Opus
+	encoder *rtpsimpleaudio.Encoder
+	decoder *rtpsimpleaudio.Decoder
+}
+
+func newFormatProcessorOpus(
+	forma *format.Opus,
+	allocateEncoder bool,
+) (*formatProcessorOpus, error) {
+	t := &formatProcessorOpus{
+		format: forma,
+	}
+
+	if allocateEncoder {
+		t.encoder = forma.CreateEncoder()
+	}
+
+	return t, nil
+}
+
+func (t *formatProcessorOpus) process(dat data, hasNonRTSPReaders bool) error { //nolint:dupl
+	tdata := dat.(*dataOpus)
+
+	if tdata.rtpPackets != nil {
+		pkt := tdata.rtpPackets[0]
+
+		// remove padding
+		pkt.Header.Padding = false
+		pkt.PaddingSize = 0
+
+		if pkt.MarshalSize() > maxPacketSize {
+			return fmt.Errorf("payload size (%d) is greater than maximum allowed (%d)",
+				pkt.MarshalSize(), maxPacketSize)
+		}
+
+		// decode from RTP
+		if hasNonRTSPReaders {
+			if t.decoder == nil {
+				t.decoder = t.format.CreateDecoder()
+			}
+
+			frame, pts, err := t.decoder.Decode(pkt)
+			if err != nil {
+				return err
+			}
+
+			tdata.frame = frame
+			tdata.pts = pts
+		}
+
+		// route packet as is
+		return nil
+	}
+
+	pkt, err := t.encoder.Encode(tdata.frame, tdata.pts)
+	if err != nil {
+		return err
+	}
+
+	tdata.rtpPackets = []*rtp.Packet{pkt}
+	return nil
+}

internal/core/formatprocessor_vp8.go

@@ -1,4 +1,4 @@
-package core
+package core //nolint:dupl
 
 import (
 	"fmt"

internal/core/formatprocessor_vp9.go

@@ -1,4 +1,4 @@
-package core
+package core //nolint:dupl
 
 import (
 	"fmt"

internal/core/hls_index.html

@@ -20,7 +20,7 @@ html, body {
 
 <video id="video" muted controls autoplay playsinline></video>
 
-<script src="https://cdn.jsdelivr.net/npm/hls.js@1.1.5"></script>
+<script src="https://cdn.jsdelivr.net/npm/hls.js@1.2.9"></script>
 
 <script>
 

internal/core/hls_muxer.go

@@ -247,20 +247,81 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 
 	var medias media.Medias
 
-	var videoFormat format.Format
-	var videoMedia *media.Media
+	videoMedia, videoFormat := m.setupVideoMedia(res.stream)
+	if videoMedia != nil {
+		medias = append(medias, videoMedia)
+	}
+
+	audioMedia, audioFormat := m.setupAudioMedia(res.stream)
+	if audioMedia != nil {
+		medias = append(medias, audioMedia)
+	}
+
+	defer res.stream.readerRemove(m)
+
+	if medias == nil {
+		return fmt.Errorf(
+			"the stream doesn't contain any supported codec (which are currently H264, H265, MPEG4-Audio, Opus)")
+	}
+
+	var err error
+	m.muxer, err = hls.NewMuxer(
+		hls.MuxerVariant(m.hlsVariant),
+		m.hlsSegmentCount,
+		time.Duration(m.hlsSegmentDuration),
+		time.Duration(m.hlsPartDuration),
+		uint64(m.hlsSegmentMaxSize),
+		videoFormat,
+		audioFormat,
+	)
+	if err != nil {
+		return fmt.Errorf("muxer error: %v", err)
+	}
+	defer m.muxer.Close()
+
+	innerReady <- struct{}{}
+
+	m.log(logger.Info, "is converting into HLS, %s",
+		sourceMediaInfo(medias))
+
+	writerDone := make(chan error)
+	go func() {
+		writerDone <- m.runWriter()
+	}()
+
+	closeCheckTicker := time.NewTicker(closeCheckPeriod)
+	defer closeCheckTicker.Stop()
 
+	for {
+		select {
+		case <-closeCheckTicker.C:
+			t := time.Unix(0, atomic.LoadInt64(m.lastRequestTime))
+			if m.remoteAddr != "" && time.Since(t) >= closeAfterInactivity {
+				m.ringBuffer.Close()
+				<-writerDone
+				return fmt.Errorf("not used anymore")
+			}
+
+		case err := <-writerDone:
+			return err
+
+		case <-innerCtx.Done():
+			m.ringBuffer.Close()
+			<-writerDone
+			return fmt.Errorf("terminated")
+		}
+	}
+}
+
+func (m *hlsMuxer) setupVideoMedia(stream *stream) (*media.Media, format.Format) {
 	var videoFormatH265 *format.H265
-	videoMedia = res.stream.medias().FindFormat(&videoFormatH265)
+	videoMedia := stream.medias().FindFormat(&videoFormatH265)
 
 	if videoFormatH265 != nil {
-		videoFormat = videoFormatH265
-		medias = append(medias, videoMedia)
-
 		videoStartPTSFilled := false
 		var videoStartPTS time.Duration
 
-		res.stream.readerAdd(m, videoMedia, videoFormat, func(dat data) {
+		stream.readerAdd(m, videoMedia, videoFormatH265, func(dat data) {
 			m.ringBuffer.Push(func() error {
 				tdata := dat.(*dataH265)
 
@@ -282,52 +343,55 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 				return nil
 			})
 		})
-	} else {
-		var videoFormatH264 *format.H264
-		videoMedia = res.stream.medias().FindFormat(&videoFormatH264)
 
-		if videoFormatH264 != nil {
-			videoFormat = videoFormatH264
-			medias = append(medias, videoMedia)
+		return videoMedia, videoFormatH265
+	}
 
-			videoStartPTSFilled := false
-			var videoStartPTS time.Duration
+	var videoFormatH264 *format.H264
+	videoMedia = stream.medias().FindFormat(&videoFormatH264)
 
-			res.stream.readerAdd(m, videoMedia, videoFormat, func(dat data) {
-				m.ringBuffer.Push(func() error {
-					tdata := dat.(*dataH264)
+	if videoFormatH264 != nil {
+		videoStartPTSFilled := false
+		var videoStartPTS time.Duration
 
-					if tdata.au == nil {
-						return nil
-					}
+		stream.readerAdd(m, videoMedia, videoFormatH264, func(dat data) {
+			m.ringBuffer.Push(func() error {
+				tdata := dat.(*dataH264)
 
-					if !videoStartPTSFilled {
-						videoStartPTSFilled = true
-						videoStartPTS = tdata.pts
-					}
-					pts := tdata.pts - videoStartPTS
+				if tdata.au == nil {
+					return nil
+				}
 
-					err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
-					if err != nil {
-						return fmt.Errorf("muxer error: %v", err)
-					}
+				if !videoStartPTSFilled {
+					videoStartPTSFilled = true
+					videoStartPTS = tdata.pts
+				}
+				pts := tdata.pts - videoStartPTS
 
-					return nil
-				})
+				err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
+				if err != nil {
+					return fmt.Errorf("muxer error: %v", err)
+				}
+
+				return nil
 			})
-		}
+		})
+
+		return videoMedia, videoFormatH264
 	}
 
-	var audioFormat *format.MPEG4Audio
-	audioMedia := res.stream.medias().FindFormat(&audioFormat)
+	return nil, nil
+}
 
-	if audioFormat != nil {
-		medias = append(medias, audioMedia)
+func (m *hlsMuxer) setupAudioMedia(stream *stream) (*media.Media, format.Format) {
+	var audioFormatMPEG4Audio *format.MPEG4Audio
+	audioMedia := stream.medias().FindFormat(&audioFormatMPEG4Audio)
 
+	if audioFormatMPEG4Audio != nil {
 		audioStartPTSFilled := false
 		var audioStartPTS time.Duration
 
-		res.stream.readerAdd(m, audioMedia, audioFormat, func(dat data) {
+		stream.readerAdd(m, audioMedia, audioFormatMPEG4Audio, func(dat data) {
 			m.ringBuffer.Push(func() error {
 				tdata := dat.(*dataMPEG4Audio)
 
@@ -342,10 +406,10 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 				pts := tdata.pts - audioStartPTS
 
 				for i, au := range tdata.aus {
-					err := m.muxer.WriteAAC(
+					err := m.muxer.WriteAudio(
 						tdata.ntp,
 						pts+time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
-							time.Second/time.Duration(audioFormat.ClockRate()),
+							time.Second/time.Duration(audioFormatMPEG4Audio.ClockRate()),
 						au)
 					if err != nil {
 						return fmt.Errorf("muxer error: %v", err)
@@ -355,61 +419,43 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 				return nil
 			})
 		})
-	}
-
-	defer res.stream.readerRemove(m)
 
-	if medias == nil {
-		return fmt.Errorf("the stream doesn't contain a supported video or audio track")
+		return audioMedia, audioFormatMPEG4Audio
 	}
 
-	var err error
-	m.muxer, err = hls.NewMuxer(
-		hls.MuxerVariant(m.hlsVariant),
-		m.hlsSegmentCount,
-		time.Duration(m.hlsSegmentDuration),
-		time.Duration(m.hlsPartDuration),
-		uint64(m.hlsSegmentMaxSize),
-		videoFormat,
-		audioFormat,
-	)
-	if err != nil {
-		return fmt.Errorf("muxer error: %v", err)
-	}
-	defer m.muxer.Close()
+	var audioFormatOpus *format.Opus
+	audioMedia = stream.medias().FindFormat(&audioFormatOpus)
 
-	innerReady <- struct{}{}
-
-	m.log(logger.Info, "is converting into HLS, %s",
-		sourceMediaInfo(medias))
+	if audioFormatOpus != nil {
+		audioStartPTSFilled := false
+		var audioStartPTS time.Duration
 
-	writerDone := make(chan error)
-	go func() {
-		writerDone <- m.runWriter()
-	}()
+		stream.readerAdd(m, audioMedia, audioFormatOpus, func(dat data) {
+			m.ringBuffer.Push(func() error {
+				tdata := dat.(*dataOpus)
 
-	closeCheckTicker := time.NewTicker(closeCheckPeriod)
-	defer closeCheckTicker.Stop()
+				if !audioStartPTSFilled {
+					audioStartPTSFilled = true
+					audioStartPTS = tdata.pts
+				}
+				pts := tdata.pts - audioStartPTS
 
-	for {
-		select {
-		case <-closeCheckTicker.C:
-			t := time.Unix(0, atomic.LoadInt64(m.lastRequestTime))
-			if m.remoteAddr != "" && time.Since(t) >= closeAfterInactivity {
-				m.ringBuffer.Close()
-				<-writerDone
-				return fmt.Errorf("not used anymore")
-			}
+				err := m.muxer.WriteAudio(
+					tdata.ntp,
+					pts,
+					tdata.frame)
+				if err != nil {
+					return fmt.Errorf("muxer error: %v", err)
+				}
 
-		case err := <-writerDone:
-			return err
+				return nil
+			})
+		})
 
-		case <-innerCtx.Done():
-			m.ringBuffer.Close()
-			<-writerDone
-			return fmt.Errorf("terminated")
-		}
+		return audioMedia, audioFormatOpus
 	}
+
+	return nil, nil
 }
 
 func (m *hlsMuxer) runWriter() error {

internal/core/rtsp_session.go

@@ -363,6 +363,17 @@ func (s *rtspSession) onRecord(ctx *gortsplib.ServerHandlerOnRecordCtx) (*base.R
 					}
 				})
 
+			case *format.Opus:
+				ctx.Session.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
+					err := s.stream.writeData(cmedia, cformat, &dataOpus{
+						rtpPackets: []*rtp.Packet{pkt},
+						ntp:        time.Now(),
+					})
+					if err != nil {
+						s.log(logger.Warn, "%v", err)
+					}
+				})
+
 			default:
 				ctx.Session.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
 					err := s.stream.writeData(cmedia, cformat, &dataGeneric{

internal/core/rtsp_source.go

@@ -203,6 +203,17 @@ func (s *rtspSource) run(ctx context.Context) error {
 							}
 						})
 
+					case *format.Opus:
+						c.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
+							err := res.stream.writeData(cmedia, cformat, &dataOpus{
+								rtpPackets: []*rtp.Packet{pkt},
+								ntp:        time.Now(),
+							})
+							if err != nil {
+								s.Log(logger.Warn, "%v", err)
+							}
+						})
+
 					default:
 						c.OnPacketRTP(medi, forma, func(pkt *rtp.Packet) {
 							err := res.stream.writeData(cmedia, cformat, &dataGeneric{

internal/hls/fmp4/init_track.go

@@ -16,30 +16,34 @@ type InitTrack struct {
 
 func (track *InitTrack) marshal(w *mp4Writer) error {
 	/*
-	   trak
-	   - tkhd
-	   - mdia
-	     - mdhd
-	     - hdlr
-	     - minf
-	       - vmhd (video only)
-	       - smhd (audio only)
-	       - dinf
-	         - dref
-	           - url
-	       - stbl
-	         - stsd
-	           - avc1 (h264 only)
-	             - avcC
-	             - pasp
-	             - btrt
-	           - mp4a (mpeg4audio only)
-	             - esds
-	             - btrt
-	         - stts
-	         - stsc
-	         - stsz
-	         - stco
+		   trak
+		   - tkhd
+		   - mdia
+			 - mdhd
+			 - hdlr
+			 - minf
+			   - vmhd (video)
+			   - smhd (audio)
+			   - dinf
+				 - dref
+				   - url
+			   - stbl
+				 - stsd
+				   - avc1 (h264)
+					 - avcC
+					 - btrt
+				   - hev1 (h265)
+					 - hvcC
+				   - mp4a (mpeg4audio)
+					 - esds
+					 - btrt
+				   - Opus (opus)
+					 - dOps
+					 - btrt
+				 - stts
+				 - stsc
+				 - stsz
+				 - stco
 	*/
 
 	_, err := w.writeBoxStart(&gomp4.Trak{}) // <trak>
@@ -72,19 +76,6 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 		width = h264SPSP.Width()
 		height = h264SPSP.Height()
 
-		_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
-			FullBox: gomp4.FullBox{
-				Flags: [3]byte{0, 0, 3},
-			},
-			TrackID: uint32(track.ID),
-			Width:   uint32(width * 65536),
-			Height:  uint32(height * 65536),
-			Matrix:  [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
-		})
-		if err != nil {
-			return err
-		}
-
 	case *format.H265:
 		h265VPS = ttrack.SafeVPS()
 		h265SPS = ttrack.SafeSPS()
@@ -97,7 +88,10 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 
 		width = h265SPSP.Width()
 		height = h265SPSP.Height()
+	}
 
+	switch track.Format.(type) {
+	case *format.H264, *format.H265:
 		_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
 			FullBox: gomp4.FullBox{
 				Flags: [3]byte{0, 0, 3},
@@ -111,7 +105,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 			return err
 		}
 
-	case *format.MPEG4Audio:
+	case *format.MPEG4Audio, *format.Opus:
 		_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
 			FullBox: gomp4.FullBox{
 				Flags: [3]byte{0, 0, 3},
@@ -149,7 +143,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 			return err
 		}
 
-	case *format.MPEG4Audio:
+	case *format.MPEG4Audio, *format.Opus:
 		_, err = w.WriteBox(&gomp4.Hdlr{ // <hdlr/>
 			HandlerType: [4]byte{'s', 'o', 'u', 'n'},
 			Name:        "SoundHandler",
@@ -175,7 +169,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 			return err
 		}
 
-	case *format.MPEG4Audio:
+	case *format.MPEG4Audio, *format.Opus:
 		_, err = w.WriteBox(&gomp4.Smhd{ // <smhd/>
 		})
 		if err != nil {
@@ -391,10 +385,6 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 		enc, _ := ttrack.Config.Marshal()
 
 		_, err = w.WriteBox(&gomp4.Esds{ // <esds/>
-			FullBox: gomp4.FullBox{
-				Version: 0,
-				Flags:   [3]byte{0x00, 0x00, 0x00},
-			},
 			Descriptors: []gomp4.Descriptor{
 				{
 					Tag:  gomp4.ESDescrTag,
@@ -443,6 +433,44 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 		if err != nil {
 			return err
 		}
+
+	case *format.Opus:
+		_, err = w.writeBoxStart(&gomp4.AudioSampleEntry{ // <Opus>
+			SampleEntry: gomp4.SampleEntry{
+				AnyTypeBox: gomp4.AnyTypeBox{
+					Type: BoxTypeOpus(),
+				},
+				DataReferenceIndex: 1,
+			},
+			ChannelCount: uint16(ttrack.ChannelCount),
+			SampleSize:   16,
+			SampleRate:   uint32(ttrack.ClockRate() * 65536),
+		})
+		if err != nil {
+			return err
+		}
+
+		_, err = w.WriteBox(&DOps{ // <dOps/>
+			OutputChannelCount: uint8(ttrack.ChannelCount),
+			PreSkip:            312,
+			InputSampleRate:    uint32(ttrack.ClockRate()),
+		})
+		if err != nil {
+			return err
+		}
+
+		_, err = w.WriteBox(&gomp4.Btrt{ // <btrt/>
+			MaxBitrate: 128825,
+			AvgBitrate: 128825,
+		})
+		if err != nil {
+			return err
+		}
+
+		err = w.writeBoxEnd() // </Opus>
+		if err != nil {
+			return err
+		}
 	}
 
 	err = w.writeBoxEnd() // </stsd>

internal/hls/fmp4/opus.go

@@ -0,0 +1,53 @@
+//nolint:gochecknoinits,revive,gocritic
+package fmp4
+
+import (
+	gomp4 "github.com/abema/go-mp4"
+)
+
+func BoxTypeOpus() gomp4.BoxType { return gomp4.StrToBoxType("Opus") }
+
+func init() {
+	gomp4.AddAnyTypeBoxDef(&gomp4.AudioSampleEntry{}, BoxTypeOpus())
+}
+
+func BoxTypeDOps() gomp4.BoxType { return gomp4.StrToBoxType("dOps") }
+
+func init() {
+	gomp4.AddBoxDef(&DOps{})
+}
+
+type DOpsChannelMappingTable struct{}
+
+type DOps struct {
+	gomp4.Box
+	Version              uint8   `mp4:"0,size=8"`
+	OutputChannelCount   uint8   `mp4:"1,size=8"`
+	PreSkip              uint16  `mp4:"2,size=16"`
+	InputSampleRate      uint32  `mp4:"3,size=32"`
+	OutputGain           int16   `mp4:"4,size=16"`
+	ChannelMappingFamily uint8   `mp4:"5,size=8"`
+	StreamCount          uint8   `mp4:"6,opt=dynamic,size=8"`
+	CoupledCount         uint8   `mp4:"7,opt=dynamic,size=8"`
+	ChannelMapping       []uint8 `mp4:"8,opt=dynamic,size=8,len=dynamic"`
+}
+
+func (DOps) GetType() gomp4.BoxType {
+	return BoxTypeDOps()
+}
+
+func (dops DOps) IsOptFieldEnabled(name string, ctx gomp4.Context) bool {
+	switch name {
+	case "StreamCount", "CoupledCount", "ChannelMapping":
+		return dops.ChannelMappingFamily != 0
+	}
+	return false
+}
+
+func (ops DOps) GetFieldLength(name string, ctx gomp4.Context) uint {
+	switch name {
+	case "ChannelMapping":
+		return uint(ops.OutputChannelCount)
+	}
+	return 0
+}

internal/hls/muxer.go

@@ -29,7 +29,7 @@ func NewMuxer(
 	partDuration time.Duration,
 	segmentMaxSize uint64,
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 ) (*Muxer, error) {
 	m := &Muxer{}
 
@@ -85,9 +85,9 @@ func (m *Muxer) WriteH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
 	return m.variant.writeH26x(ntp, pts, au)
 }
 
-// WriteAAC writes an AAC access unit.
-func (m *Muxer) WriteAAC(ntp time.Time, pts time.Duration, au []byte) error {
-	return m.variant.writeAAC(ntp, pts, au)
+// WriteAudio writes an audio access unit.
+func (m *Muxer) WriteAudio(ntp time.Time, pts time.Duration, au []byte) error {
+	return m.variant.writeAudio(ntp, pts, au)
 }
 
 // File returns a file reader.

internal/hls/muxer_primary_playlist.go

@@ -31,6 +31,9 @@ func codecParameters(track format.Format) string {
 	case *format.MPEG4Audio:
 		// https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
 		return "mp4a.40." + strconv.FormatInt(int64(ttrack.Config.Type), 10)
+
+	case *format.Opus:
+		return "opus"
 	}
 
 	return ""
@@ -39,13 +42,13 @@ func codecParameters(track format.Format) string {
 type muxerPrimaryPlaylist struct {
 	fmp4       bool
 	videoTrack format.Format
-	audioTrack *format.MPEG4Audio
+	audioTrack format.Format
 }
 
 func newMuxerPrimaryPlaylist(
 	fmp4 bool,
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 ) *muxerPrimaryPlaylist {
 	return &muxerPrimaryPlaylist{
 		fmp4:       fmp4,

internal/hls/muxer_test.go

@@ -75,13 +75,13 @@ func TestMuxerVideoAudio(t *testing.T) {
 			require.NoError(t, err)
 
 			d = 3 * time.Second
-			err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{
+			err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
 				0x01, 0x02, 0x03, 0x04,
 			})
 			require.NoError(t, err)
 
 			d = 3500 * time.Millisecond
-			err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{
+			err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
 				0x01, 0x02, 0x03, 0x04,
 			})
 			require.NoError(t, err)
@@ -94,7 +94,7 @@ func TestMuxerVideoAudio(t *testing.T) {
 			require.NoError(t, err)
 
 			d = 4500 * time.Millisecond
-			err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{
+			err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
 				0x01, 0x02, 0x03, 0x04,
 			})
 			require.NoError(t, err)
@@ -323,20 +323,20 @@ func TestMuxerAudioOnly(t *testing.T) {
 
 			for i := 0; i < 100; i++ {
 				d := 1 * time.Second
-				err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{
+				err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
 					0x01, 0x02, 0x03, 0x04,
 				})
 				require.NoError(t, err)
 			}
 
 			d := 2 * time.Second
-			err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{
+			err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
 				0x01, 0x02, 0x03, 0x04,
 			})
 			require.NoError(t, err)
 
 			d = 3 * time.Second
-			err = m.WriteAAC(testTime.Add(d-1*time.Second), d, []byte{
+			err = m.WriteAudio(testTime.Add(d-1*time.Second), d, []byte{
 				0x01, 0x02, 0x03, 0x04,
 			})
 			require.NoError(t, err)

internal/hls/muxer_variant.go

@@ -17,6 +17,6 @@ const (
 type muxerVariant interface {
 	close()
 	writeH26x(time.Time, time.Duration, [][]byte) error
-	writeAAC(time.Time, time.Duration, []byte) error
+	writeAudio(time.Time, time.Duration, []byte) error
 	file(name string, msn string, part string, skip string) *MuxerFileResponse
 }

internal/hls/muxer_variant_fmp4.go

@@ -48,7 +48,7 @@ type muxerVariantFMP4 struct {
 	playlist   *muxerVariantFMP4Playlist
 	segmenter  *muxerVariantFMP4Segmenter
 	videoTrack format.Format
-	audioTrack *format.MPEG4Audio
+	audioTrack format.Format
 
 	mutex           sync.Mutex
 	lastVideoParams [][]byte
@@ -62,7 +62,7 @@ func newMuxerVariantFMP4(
 	partDuration time.Duration,
 	segmentMaxSize uint64,
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 ) *muxerVariantFMP4 {
 	v := &muxerVariantFMP4{
 		videoTrack: videoTrack,
@@ -99,8 +99,8 @@ func (v *muxerVariantFMP4) writeH26x(ntp time.Time, pts time.Duration, au [][]by
 	return v.segmenter.writeH26x(ntp, pts, au)
 }
 
-func (v *muxerVariantFMP4) writeAAC(ntp time.Time, pts time.Duration, au []byte) error {
-	return v.segmenter.writeAAC(ntp, pts, au)
+func (v *muxerVariantFMP4) writeAudio(ntp time.Time, pts time.Duration, au []byte) error {
+	return v.segmenter.writeAudio(ntp, pts, au)
 }
 
 func (v *muxerVariantFMP4) mustRegenerateInit() bool {

internal/hls/muxer_variant_fmp4_part.go

@@ -18,7 +18,7 @@ func fmp4PartName(id uint64) string {
 
 type muxerVariantFMP4Part struct {
 	videoTrack format.Format
-	audioTrack *format.MPEG4Audio
+	audioTrack format.Format
 	id         uint64
 
 	isIndependent       bool
@@ -34,7 +34,7 @@ type muxerVariantFMP4Part struct {
 
 func newMuxerVariantFMP4Part(
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 	id uint64,
 ) *muxerVariantFMP4Part {
 	p := &muxerVariantFMP4Part{
@@ -130,7 +130,7 @@ func (p *muxerVariantFMP4Part) writeH264(sample *augmentedVideoSample) {
 	p.videoSamples = append(p.videoSamples, &sample.PartSample)
 }
 
-func (p *muxerVariantFMP4Part) writeAAC(sample *augmentedAudioSample) {
+func (p *muxerVariantFMP4Part) writeAudio(sample *augmentedAudioSample) {
 	if !p.audioStartDTSFilled {
 		p.audioStartDTSFilled = true
 		p.audioStartDTS = sample.dts

internal/hls/muxer_variant_fmp4_playlist.go

@@ -71,7 +71,7 @@ type muxerVariantFMP4Playlist struct {
 	lowLatency   bool
 	segmentCount int
 	videoTrack   format.Format
-	audioTrack   *format.MPEG4Audio
+	audioTrack   format.Format
 
 	mutex              sync.Mutex
 	cond               *sync.Cond
@@ -90,7 +90,7 @@ func newMuxerVariantFMP4Playlist(
 	lowLatency bool,
 	segmentCount int,
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 ) *muxerVariantFMP4Playlist {
 	p := &muxerVariantFMP4Playlist{
 		lowLatency:     lowLatency,

internal/hls/muxer_variant_fmp4_segment.go

@@ -46,7 +46,7 @@ type muxerVariantFMP4Segment struct {
 	startDTS        time.Duration
 	segmentMaxSize  uint64
 	videoTrack      format.Format
-	audioTrack      *format.MPEG4Audio
+	audioTrack      format.Format
 	genPartID       func() uint64
 	onPartFinalized func(*muxerVariantFMP4Part)
 
@@ -64,7 +64,7 @@ func newMuxerVariantFMP4Segment(
 	startDTS time.Duration,
 	segmentMaxSize uint64,
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 	genPartID func() uint64,
 	onPartFinalized func(*muxerVariantFMP4Part),
 ) *muxerVariantFMP4Segment {
@@ -155,14 +155,14 @@ func (s *muxerVariantFMP4Segment) writeH264(sample *augmentedVideoSample, adjust
 	return nil
 }
 
-func (s *muxerVariantFMP4Segment) writeAAC(sample *augmentedAudioSample, adjustedPartDuration time.Duration) error {
+func (s *muxerVariantFMP4Segment) writeAudio(sample *augmentedAudioSample, adjustedPartDuration time.Duration) error {
 	size := uint64(len(sample.Payload))
 	if (s.size + size) > s.segmentMaxSize {
 		return fmt.Errorf("reached maximum segment size")
 	}
 	s.size += size
 
-	s.currentPart.writeAAC(sample)
+	s.currentPart.writeAudio(sample)
 
 	// switch part
 	if s.lowLatency && s.videoTrack == nil &&

internal/hls/muxer_variant_fmp4_segmenter.go

@@ -79,7 +79,7 @@ type muxerVariantFMP4Segmenter struct {
 	partDuration       time.Duration
 	segmentMaxSize     uint64
 	videoTrack         format.Format
-	audioTrack         *format.MPEG4Audio
+	audioTrack         format.Format
 	onSegmentFinalized func(*muxerVariantFMP4Segment)
 	onPartFinalized    func(*muxerVariantFMP4Part)
 
@@ -104,7 +104,7 @@ func newMuxerVariantFMP4Segmenter(
 	partDuration time.Duration,
 	segmentMaxSize uint64,
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 	onSegmentFinalized func(*muxerVariantFMP4Segment),
 	onPartFinalized func(*muxerVariantFMP4Part),
 ) *muxerVariantFMP4Segmenter {
@@ -317,7 +317,7 @@ func (m *muxerVariantFMP4Segmenter) writeH26xEntry(
 	return nil
 }
 
-func (m *muxerVariantFMP4Segmenter) writeAAC(ntp time.Time, dts time.Duration, au []byte) error {
+func (m *muxerVariantFMP4Segmenter) writeAudio(ntp time.Time, dts time.Duration, au []byte) error {
 	if m.videoTrack != nil {
 		// wait for the video track
 		if !m.videoFirstRandomAccessReceived {
@@ -367,7 +367,7 @@ func (m *muxerVariantFMP4Segmenter) writeAAC(ntp time.Time, dts time.Duration, a
 		}
 	}
 
-	err := m.currentSegment.writeAAC(sample, m.partDuration)
+	err := m.currentSegment.writeAudio(sample, m.partDuration)
 	if err != nil {
 		return err
 	}

internal/hls/muxer_variant_mpegts.go

@@ -17,7 +17,7 @@ func newMuxerVariantMPEGTS(
 	segmentDuration time.Duration,
 	segmentMaxSize uint64,
 	videoTrack format.Format,
-	audioTrack *format.MPEG4Audio,
+	audioTrack format.Format,
 ) (*muxerVariantMPEGTS, error) {
 	var videoTrackH264 *format.H264
 	if videoTrack != nil {
@@ -25,7 +25,17 @@ func newMuxerVariantMPEGTS(
 		videoTrackH264, ok = videoTrack.(*format.H264)
 		if !ok {
 			return nil, fmt.Errorf(
-				"the MPEG-TS variant of HLS doesn't support H265. Use the fMP4 or Low-Latency variants instead")
+				"the MPEG-TS variant of HLS only supports H264 video. Use the fMP4 or Low-Latency variants instead")
+		}
+	}
+
+	var audioTrackMPEG4Audio *format.MPEG4Audio
+	if audioTrack != nil {
+		var ok bool
+		audioTrackMPEG4Audio, ok = audioTrack.(*format.MPEG4Audio)
+		if !ok {
+			return nil, fmt.Errorf(
+				"the MPEG-TS variant of HLS only supports MPEG4-audio. Use the fMP4 or Low-Latency variants instead")
 		}
 	}
 
@@ -37,7 +47,7 @@ func newMuxerVariantMPEGTS(
 		segmentDuration,
 		segmentMaxSize,
 		videoTrackH264,
-		audioTrack,
+		audioTrackMPEG4Audio,
 		func(seg *muxerVariantMPEGTSSegment) {
 			v.playlist.pushSegment(seg)
 		},
@@ -54,7 +64,7 @@ func (v *muxerVariantMPEGTS) writeH26x(ntp time.Time, pts time.Duration, nalus [
 	return v.segmenter.writeH264(ntp, pts, nalus)
 }
 
-func (v *muxerVariantMPEGTS) writeAAC(ntp time.Time, pts time.Duration, au []byte) error {
+func (v *muxerVariantMPEGTS) writeAudio(ntp time.Time, pts time.Duration, au []byte) error {
 	return v.segmenter.writeAAC(ntp, pts, au)
 }