hls: support reading and proxying AV1 tracks (#2155)

README.md

@@ -28,7 +28,7 @@ Live streams can be published to the server with:
 |[RTSP cameras and servers](#rtsp-cameras-and-servers)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G726, G722, G711, LPCM and any RTP-compatible codec|
 |[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3)|
 |[RTMP cameras and servers](#rtmp-cameras-and-servers)|RTMP, RTMPS, Enhanced RTMP|H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3)|
-|[HLS cameras and servers](#hls-cameras-and-servers)|Low-Latency HLS, MP4-based HLS, legacy HLS|H265, H264|Opus, MPEG-4 Audio (AAC)|
+|[HLS cameras and servers](#hls-cameras-and-servers)|Low-Latency HLS, MP4-based HLS, legacy HLS|AV1, H265, H264|Opus, MPEG-4 Audio (AAC)|
 |[UDP/MPEG-TS](#udpmpeg-ts)|Unicast, broadcast, multicast|H265, H264|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3)|
 |[Raspberry Pi Cameras](#raspberry-pi-cameras)||H264||
 
@@ -40,7 +40,7 @@ And can be read from the server with:
 |[WebRTC](#webrtc)|Browser-based, WHEP|AV1, VP9, VP8, H264|Opus, G722, G711|
 |[RTSP](#rtsp)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G726, G722, G711, LPCM and any RTP-compatible codec|
 |[RTMP](#rtmp)|RTMP, RTMPS, Enhanced RTMP|H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3)|
-|[HLS](#hls)|Low-Latency HLS, MP4-based HLS, legacy HLS|H265, H264|Opus, MPEG-4 Audio (AAC)|
+|[HLS](#hls)|Low-Latency HLS, MP4-based HLS, legacy HLS|AV1, H265, H264|Opus, MPEG-4 Audio (AAC)|
 
 **Features**
 

go.mod

@@ -6,9 +6,9 @@ require (
 	code.cloudfoundry.org/bytefmt v0.0.0
 	github.com/abema/go-mp4 v0.11.0
 	github.com/alecthomas/kong v0.8.0
-	github.com/bluenviron/gohlslib v0.3.1-0.20230805121442-9577da1cb068
+	github.com/bluenviron/gohlslib v0.3.1-0.20230806000705-4ca609772363
-	github.com/bluenviron/gortsplib/v3 v3.9.1-0.20230805122812-17acec3f0dea
+	github.com/bluenviron/gortsplib/v3 v3.9.1-0.20230806001215-926a452fb745
-	github.com/bluenviron/mediacommon v0.7.1-0.20230805114828-bee33f3b286d
+	github.com/bluenviron/mediacommon v0.7.1-0.20230805234008-34d20294a26b
 	github.com/datarhei/gosrt v0.5.3
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/gin-gonic/gin v1.9.1
@@ -31,7 +31,7 @@ require (
 )
 
 require (
-	github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 // indirect
+	github.com/aler9/writerseeker v1.0.0 // indirect
 	github.com/asticode/go-astikit v0.30.0 // indirect
 	github.com/asticode/go-astits v1.12.0 // indirect
 	github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c // indirect

go.sum

@@ -6,20 +6,20 @@ github.com/alecthomas/kong v0.8.0/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqr
 github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
 github.com/aler9/gosrt v0.0.0-20230803201410-b26769a3a222 h1:69Oj9CKfxxkTqFtBfBNo7zP4HcxEwjpFnFNHHX2Z06c=
 github.com/aler9/gosrt v0.0.0-20230803201410-b26769a3a222/go.mod h1:EryxR+Men7aW67IX2FEo56SU+Pay9OHCw3kZx23qKyQ=
-github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=
+github.com/aler9/writerseeker v1.0.0 h1:3wI/NtI20WqX8iLjrivym5VGXxug3EkRPaLRw6k9f9M=
-github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U=
+github.com/aler9/writerseeker v1.0.0/go.mod h1:J5H8yGtvHnTM3DddPusobve1mDTYgdAj7eAwjGj8FXY=
 github.com/asticode/go-astikit v0.30.0 h1:DkBkRQRIxYcknlaU7W7ksNfn4gMFsB0tqMJflxkRsZA=
 github.com/asticode/go-astikit v0.30.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0=
 github.com/asticode/go-astits v1.12.0 h1:BiefTgVEyPgEB8nT6J+Sys/uxE4H/a04SW/aedpOpPc=
 github.com/asticode/go-astits v1.12.0/go.mod h1:QSHmknZ51pf6KJdHKZHJTLlMegIrhega3LPWz3ND/iI=
 github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c h1:8XZeJrs4+ZYhJeJ2aZxADI2tGADS15AzIF8MQ8XAhT4=
 github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c/go.mod h1:x1vxHcL/9AVzuk5HOloOEPrtJY0MaalYr78afXZ+pWI=
-github.com/bluenviron/gohlslib v0.3.1-0.20230805121442-9577da1cb068 h1:mNSdk14esA9ggCP0+gDmcG9wLvb7QzHLta071nTLYe4=
+github.com/bluenviron/gohlslib v0.3.1-0.20230806000705-4ca609772363 h1:ysN/BZSY9Q7Ditvro4zvSLLYyxTPyiiVl0ALeY/rHHQ=
-github.com/bluenviron/gohlslib v0.3.1-0.20230805121442-9577da1cb068/go.mod h1:DouD/43mEaXgwbeQZdtVhzRnA7RuKFYeaVPz6tj/Rfk=
+github.com/bluenviron/gohlslib v0.3.1-0.20230806000705-4ca609772363/go.mod h1:2aXz4lSl2pToXk9zYmWhdhR02OiRRZCQiFdrQcIWLBc=
-github.com/bluenviron/gortsplib/v3 v3.9.1-0.20230805122812-17acec3f0dea h1:AK6GuUoXDaGfk3HDe6qMJ/Swmj1hIxVtMuw98sa2VO8=
+github.com/bluenviron/gortsplib/v3 v3.9.1-0.20230806001215-926a452fb745 h1:LR8F5ng75c9KxWLrPWj8bskXCR7AJXv8gm4duKfzUjc=
-github.com/bluenviron/gortsplib/v3 v3.9.1-0.20230805122812-17acec3f0dea/go.mod h1:9GOyv9dKyDjMHXcrh7rytIoXWvRk+/DC0Mjq3vqEyCU=
+github.com/bluenviron/gortsplib/v3 v3.9.1-0.20230806001215-926a452fb745/go.mod h1:7owCGeF8gQfziPCWL9UbL+Yqogl13gKJeRI8jhnpadg=
-github.com/bluenviron/mediacommon v0.7.1-0.20230805114828-bee33f3b286d h1:ye0ze/XtVq23NwgxV/PWcjtagcZXTKZq+V6ZBT0HoTY=
+github.com/bluenviron/mediacommon v0.7.1-0.20230805234008-34d20294a26b h1:/csYQQDmZyXw0CVATJSPBkwT3JYap418W7LLX7mscxw=
-github.com/bluenviron/mediacommon v0.7.1-0.20230805114828-bee33f3b286d/go.mod h1:8Y0rvMJDUCgqDegYrUxG1rbumB6DV0TZ98Rw9mowIrA=
+github.com/bluenviron/mediacommon v0.7.1-0.20230805234008-34d20294a26b/go.mod h1:LR4w8cpvzo2ZcmBwXcentvBj7ZlyF9g9xP4dDbt8uJw=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
 github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=

internal/core/hls_index.html

@@ -21,7 +21,7 @@ html, body {
 
 <video id="video" muted controls autoplay playsinline></video>
 
-<script src="https://cdn.jsdelivr.net/npm/hls.js@1.4.3"></script>
+<script src="https://cdn.jsdelivr.net/npm/hls.js@1.4.10"></script>
 
 <script>
 

internal/core/hls_manager_test.go

@@ -165,7 +165,7 @@ func TestHLSRead(t *testing.T) {
 		"#EXT-X-VERSION:9\n"+
 		"#EXT-X-INDEPENDENT-SEGMENTS\n"+
 		"\n"+
-		"#EXT-X-STREAM-INF:BANDWIDTH=1256,AVERAGE-BANDWIDTH=1256,"+
+		"#EXT-X-STREAM-INF:BANDWIDTH=1192,AVERAGE-BANDWIDTH=1192,"+
 		"CODECS=\"avc1.42c028\",RESOLUTION=1920x1080,FRAME-RATE=30.000\n"+
 		"stream.m3u8\n", string(cnt))
 

internal/core/hls_muxer.go

@@ -336,8 +336,43 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 }
 
 func (m *hlsMuxer) createVideoTrack(stream *stream.Stream) (*media.Media, *gohlslib.Track) {
+	var videoFormatAV1 *formats.AV1
+	videoMedia := stream.Medias().FindFormat(&videoFormatAV1)
+
+	if videoFormatAV1 != nil {
+		startPTSFilled := false
+		var startPTS time.Duration
+
+		stream.AddReader(m, videoMedia, videoFormatAV1, func(unit formatprocessor.Unit) {
+			m.ringBuffer.Push(func() error {
+				tunit := unit.(*formatprocessor.UnitAV1)
+
+				if tunit.OBUs == nil {
+					return nil
+				}
+
+				if !startPTSFilled {
+					startPTSFilled = true
+					startPTS = tunit.PTS
+				}
+
+				pts := tunit.PTS - startPTS
+				err := m.muxer.WriteAV1(tunit.NTP, pts, tunit.OBUs)
+				if err != nil {
+					return fmt.Errorf("muxer error: %v", err)
+				}
+
+				return nil
+			})
+		})
+
+		return videoMedia, &gohlslib.Track{
+			Codec: &codecs.AV1{},
+		}
+	}
+
 	var videoFormatH265 *formats.H265
-	videoMedia := stream.Medias().FindFormat(&videoFormatH265)
+	videoMedia = stream.Medias().FindFormat(&videoFormatH265)
 
 	if videoFormatH265 != nil {
 		startPTSFilled := false
@@ -535,7 +570,7 @@ func (m *hlsMuxer) createAudioTrack(stream *stream.Stream) (*media.Media, *gohls
 
 		return audioMedia, &gohlslib.Track{
 			Codec: &codecs.Opus{
-				Channels: func() int {
+				ChannelCount: func() int {
 					if audioFormatOpus.IsStereo {
 						return 2
 					}

internal/core/hls_source.go

@@ -145,7 +145,7 @@ func (s *hlsSource) run(ctx context.Context, cnf *conf.PathConf, reloadConf chan
 					Type: media.TypeAudio,
 					Formats: []formats.Format{&formats.Opus{
 						PayloadTyp: 96,
-						IsStereo:   (tcodec.Channels == 2),
+						IsStereo:   (tcodec.ChannelCount == 2),
 					}},
 				}
 

internal/rtmp/boxes_av1.go

@@ -1,37 +0,0 @@
-package rtmp
-
-import (
-	gomp4 "github.com/abema/go-mp4"
-)
-
-// BoxTypeAv1C returns the box type.
-func BoxTypeAv1C() gomp4.BoxType { return gomp4.StrToBoxType("av1C") }
-
-func init() { //nolint:gochecknoinits
-	gomp4.AddBoxDef(&Av1C{})
-}
-
-// Av1C is a Av1C ISO-BMFF box.
-type Av1C struct {
-	gomp4.Box
-	Marker                           uint8   `mp4:"0,size=1,const=1"`
-	Version                          uint8   `mp4:"1,size=7,const=1"`
-	SeqProfile                       uint8   `mp4:"2,size=3"`
-	SeqLevelIdx0                     uint8   `mp4:"3,size=5"`
-	SeqTier0                         uint8   `mp4:"4,size=1"`
-	HighBitdepth                     uint8   `mp4:"5,size=1"`
-	TwelveBit                        uint8   `mp4:"6,size=1"`
-	Monochrome                       uint8   `mp4:"7,size=1"`
-	ChromaSubsamplingX               uint8   `mp4:"8,size=1"`
-	ChromaSubsamplingY               uint8   `mp4:"9,size=1"`
-	ChromaSamplePosition             uint8   `mp4:"10,size=2"`
-	Reserved                         uint8   `mp4:"11,size=3,const=0"`
-	InitialPresentationDelayPresent  uint8   `mp4:"12,size=1"`
-	InitialPresentationDelayMinusOne uint8   `mp4:"13,size=4"`
-	ConfigOBUs                       []uint8 `mp4:"14,size=8"`
-}
-
-// GetType returns the box type.
-func (Av1C) GetType() gomp4.BoxType {
-	return BoxTypeAv1C()
-}

internal/rtmp/reader.go

@@ -6,12 +6,13 @@ import (
 	"fmt"
 	"time"
 
-	gomp4 "github.com/abema/go-mp4"
+	"github.com/abema/go-mp4"
 	"github.com/bluenviron/gortsplib/v3/pkg/formats"
 	"github.com/bluenviron/mediacommon/pkg/codecs/av1"
 	"github.com/bluenviron/mediacommon/pkg/codecs/h264"
 	"github.com/bluenviron/mediacommon/pkg/codecs/h265"
 	"github.com/bluenviron/mediacommon/pkg/codecs/mpeg4audio"
+	"github.com/bluenviron/mediacommon/pkg/formats/fmp4"
 	"github.com/notedit/rtmp/format/flv/flvio"
 
 	"github.com/bluenviron/mediamtx/internal/rtmp/h264conf"
@@ -30,7 +31,7 @@ type OnDataMPEG4AudioFunc func(pts time.Duration, au []byte)
 // OnDataMPEG1AudioFunc is the prototype of the callback passed to OnDataMPEG1Audio().
 type OnDataMPEG1AudioFunc func(pts time.Duration, frame []byte)
 
-func h265FindNALU(array []gomp4.HEVCNaluArray, typ h265.NALUType) []byte {
+func h265FindNALU(array []mp4.HEVCNaluArray, typ h265.NALUType) []byte {
 	for _, entry := range array {
 		if entry.NaluType == byte(typ) && entry.NumNalus == 1 &&
 			h265.NALUType((entry.Nalus[0].NALUnit[0]>>1)&0b111111) == typ {
@@ -215,8 +216,8 @@ func tracksFromMetadata(conn *Conn, payload []interface{}) (formats.Format, form
 			if videoTrack == nil {
 				switch tmsg.FourCC {
 				case message.FourCCHEVC:
-					var hvcc gomp4.HvcC
+					var hvcc mp4.HvcC
-					_, err := gomp4.Unmarshal(bytes.NewReader(tmsg.Config), uint64(len(tmsg.Config)), &hvcc, gomp4.Context{})
+					_, err := mp4.Unmarshal(bytes.NewReader(tmsg.Config), uint64(len(tmsg.Config)), &hvcc, mp4.Context{})
 					if err != nil {
 						return nil, nil, fmt.Errorf("invalid H265 configuration: %v", err)
 					}
@@ -236,8 +237,8 @@ func tracksFromMetadata(conn *Conn, payload []interface{}) (formats.Format, form
 					}
 
 				case message.FourCCAV1:
-					var av1c Av1C
+					var av1c fmp4.Av1C
-					_, err := gomp4.Unmarshal(bytes.NewReader(tmsg.Config), uint64(len(tmsg.Config)), &av1c, gomp4.Context{})
+					_, err := mp4.Unmarshal(bytes.NewReader(tmsg.Config), uint64(len(tmsg.Config)), &av1c, mp4.Context{})
 					if err != nil {
 						return nil, nil, fmt.Errorf("invalid AV1 configuration: %v", err)
 					}