support reading H265 tracks with HLS (#1342)

* support reading H265 tracks with HLS

* update README

README.md

@@ -22,7 +22,7 @@ And can be read from the server with:
 |--------|--------|------|
 |RTSP|UDP, UDP-Multicast, TCP, RTSPS|H264, H265, VP8, VP9, AV1, MPEG2, M-JPEG, MP3, MPEG4 Audio (AAC), Opus, G711, G722, LPCM and any RTP-compatible codec|
 |RTMP|RTMP, RTMPS|H264, MPEG4 Audio (AAC)|
-|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, MPEG4 Audio (AAC)|
+|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, H265, MPEG4 Audio (AAC)|
 |WebRTC||H264, VP8, VP9, Opus, G711, G722|
 
 Features:
@@ -89,6 +89,7 @@ Features:
   * [Encryption](#encryption-1)
 * [HLS protocol](#hls-protocol)
   * [General usage](#general-usage-2)
+  * [Browser support](#browser-support)
   * [Embedding](#embedding)
   * [Low-Latency variant](#low-latency-variant)
   * [Decreasing latency](#decreasing-latency)
@@ -903,7 +904,13 @@ http://localhost:8888/mystream
 
 where `mystream` is the name of a stream that is being published.
 
-Please be aware that HLS only supports a single H264 video track and a single AAC audio track due to limitations of most browsers. If you want to use HLS with streams that use other codecs, you have to re-encode them, for instance by using _FFmpeg_:
+### Browser support
+
+Although the server can produce HLS with a variety of video and audio codecs (that are listed at the beginningo of the README), not all browsers can read all codecs. You can check what codecs your browser can read by visiting this page:
+
+https://jsfiddle.net/7nwxmLto
+
+If you want to increase the compatibility of the stream in order to support most browsers, you have to re-encode it by using the H264 and AAC codecs, for instance by using _FFmpeg_:
 
 ```
 ffmpeg -i rtsp://original-source -pix_fmt yuv420p -c:v libx264 -preset ultrafast -b:v 600k -c:a aac -b:a 160k -f rtsp rtsp://localhost:8554/mystream

go.mod

@@ -3,9 +3,9 @@ module github.com/aler9/rtsp-simple-server
 go 1.18
 
 require (
-	code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
-	github.com/abema/go-mp4 v0.8.0
-	github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562
+	code.cloudfoundry.org/bytefmt v0.0.0
+	github.com/abema/go-mp4 v0.0.0
+	github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823
 	github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gin-gonic/gin v1.8.1
@@ -68,3 +68,5 @@ require (
 replace github.com/orcaman/writerseeker => github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82
 
 replace code.cloudfoundry.org/bytefmt => github.com/cloudfoundry/bytefmt v0.0.0-20211005130812-5bb3c17173e5
+
+replace github.com/abema/go-mp4 => github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218

go.sum

@@ -1,11 +1,11 @@
-github.com/abema/go-mp4 v0.8.0 h1:JHYkOvTfBpTnqJHiFFOXe8d6wiFy5MtDnA10fgccNqY=
-github.com/abema/go-mp4 v0.8.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562 h1://BJIsHw2vYKdPL6sKbxZEnlGPpj2PTznNzRpou87ds=
-github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
+github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218 h1:Zak89uY+y0q/gL7jaKbl2XeyMOLT/5qVuW6TIJphEJY=
+github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
+github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823 h1:EFq9LqgA15drNgXj3hNlmAouxjMYb9jyyBq6hmjDO8U=
+github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
 github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=
 github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U=
 github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=

internal/core/formatprocessor_h264.go

@@ -71,7 +71,7 @@ type dataH264 struct {
 	rtpPackets []*rtp.Packet
 	ntp        time.Time
 	pts        time.Duration
-	nalus      [][]byte
+	au         [][]byte
 }
 
 func (d *dataH264) getRTPPackets() []*rtp.Packet {
@@ -134,22 +134,23 @@ func (t *formatProcessorH264) updateTrackParametersFromNALUs(nalus [][]byte) {
 	}
 }
 
-// remux is needed to fix corrupted streams and make streams
-// compatible with all protocols.
-func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
-	addSPSPPS := false
+func (t *formatProcessorH264) remuxAccessUnit(nalus [][]byte) [][]byte {
+	addParameters := false
 	n := 0
+
 	for _, nalu := range nalus {
 		typ := h264.NALUType(nalu[0] & 0x1F)
+
 		switch typ {
-		case h264.NALUTypeSPS, h264.NALUTypePPS:
+		case h264.NALUTypeSPS, h264.NALUTypePPS: // remove parameters
 			continue
-		case h264.NALUTypeAccessUnitDelimiter:
+
+		case h264.NALUTypeAccessUnitDelimiter: // remove AUDs
 			continue
-		case h264.NALUTypeIDR:
-			// prepend SPS and PPS to the group if there's at least an IDR
-			if !addSPSPPS {
-				addSPSPPS = true
+
+		case h264.NALUTypeIDR: // prepend parameters if there's at least an IDR
+			if !addParameters {
+				addParameters = true
 				n += 2
 			}
 		}
@@ -163,7 +164,7 @@ func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
 	filteredNALUs := make([][]byte, n)
 	i := 0
 
-	if addSPSPPS {
+	if addParameters {
 		filteredNALUs[0] = t.format.SafeSPS()
 		filteredNALUs[1] = t.format.SafePPS()
 		i = 2
@@ -171,13 +172,12 @@ func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
 
 	for _, nalu := range nalus {
 		typ := h264.NALUType(nalu[0] & 0x1F)
+
 		switch typ {
 		case h264.NALUTypeSPS, h264.NALUTypePPS:
-			// remove since they're automatically added
 			continue
 
 		case h264.NALUTypeAccessUnitDelimiter:
-			// remove since it is not needed
 			continue
 		}
 
@@ -227,7 +227,7 @@ func (t *formatProcessorH264) process(dat data, hasNonRTSPReaders bool) error {
 			}
 
 			// DecodeUntilMarker() is necessary, otherwise Encode() generates partial groups
-			nalus, pts, err := t.decoder.DecodeUntilMarker(pkt)
+			au, pts, err := t.decoder.DecodeUntilMarker(pkt)
 			if err != nil {
 				if err == rtph264.ErrNonStartingPacketAndNoPrevious || err == rtph264.ErrMorePacketsNeeded {
 					return nil
@@ -235,10 +235,9 @@ func (t *formatProcessorH264) process(dat data, hasNonRTSPReaders bool) error {
 				return err
 			}
 
-			tdata.nalus = nalus
+			tdata.au = au
 			tdata.pts = pts
-
-			tdata.nalus = t.remuxNALUs(tdata.nalus)
+			tdata.au = t.remuxAccessUnit(tdata.au)
 		}
 
 		// route packet as is
@@ -246,11 +245,11 @@ func (t *formatProcessorH264) process(dat data, hasNonRTSPReaders bool) error {
 			return nil
 		}
 	} else {
-		t.updateTrackParametersFromNALUs(tdata.nalus)
-		tdata.nalus = t.remuxNALUs(tdata.nalus)
+		t.updateTrackParametersFromNALUs(tdata.au)
+		tdata.au = t.remuxAccessUnit(tdata.au)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.nalus, tdata.pts)
+	pkts, err := t.encoder.Encode(tdata.au, tdata.pts)
 	if err != nil {
 		return err
 	}

internal/core/formatprocessor_h265.go

@@ -78,7 +78,7 @@ type dataH265 struct {
 	rtpPackets []*rtp.Packet
 	ntp        time.Time
 	pts        time.Duration
-	nalus      [][]byte
+	au         [][]byte
 }
 
 func (d *dataH265) getRTPPackets() []*rtp.Packet {
@@ -128,12 +128,82 @@ func (t *formatProcessorH265) updateTrackParametersFromRTPPacket(pkt *rtp.Packet
 }
 
 func (t *formatProcessorH265) updateTrackParametersFromNALUs(nalus [][]byte) {
-	// TODO: extract VPS, SPS, PPS and set them into the track
+	for _, nalu := range nalus {
+		typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+		switch typ {
+		case h265.NALUType_VPS_NUT:
+			if !bytes.Equal(nalu, t.format.SafeVPS()) {
+				t.format.SafeSetVPS(nalu)
 			}
 
-func (t *formatProcessorH265) remuxNALUs(nalus [][]byte) [][]byte {
-	// TODO: add VPS, SPS, PPS before IDRs
-	return nalus
+		case h265.NALUType_SPS_NUT:
+			if !bytes.Equal(nalu, t.format.SafePPS()) {
+				t.format.SafeSetSPS(nalu)
+			}
+
+		case h265.NALUType_PPS_NUT:
+			if !bytes.Equal(nalu, t.format.SafePPS()) {
+				t.format.SafeSetPPS(nalu)
+			}
+		}
+	}
+}
+
+func (t *formatProcessorH265) remuxAccessUnit(nalus [][]byte) [][]byte {
+	addParameters := false
+	n := 0
+
+	for _, nalu := range nalus {
+		typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+		switch typ {
+		case h265.NALUType_VPS_NUT, h265.NALUType_SPS_NUT, h265.NALUType_PPS_NUT: // remove parameters
+			continue
+
+		case h265.NALUType_AUD_NUT: // remove AUDs
+			continue
+
+		// prepend parameters if there's at least a random access unit
+		case h265.NALUType_IDR_W_RADL, h265.NALUType_IDR_N_LP, h265.NALUType_CRA_NUT:
+			if !addParameters {
+				addParameters = true
+				n += 3
+			}
+		}
+		n++
+	}
+
+	if n == 0 {
+		return nil
+	}
+
+	filteredNALUs := make([][]byte, n)
+	i := 0
+
+	if addParameters {
+		filteredNALUs[0] = t.format.SafeVPS()
+		filteredNALUs[1] = t.format.SafeSPS()
+		filteredNALUs[2] = t.format.SafePPS()
+		i = 3
+	}
+
+	for _, nalu := range nalus {
+		typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+		switch typ {
+		case h265.NALUType_VPS_NUT, h265.NALUType_SPS_NUT, h265.NALUType_PPS_NUT:
+			continue
+
+		case h265.NALUType_AUD_NUT:
+			continue
+		}
+
+		filteredNALUs[i] = nalu
+		i++
+	}
+
+	return filteredNALUs
 }
 
 func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error { //nolint:dupl
@@ -175,7 +245,7 @@ func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error {
 			}
 
 			// DecodeUntilMarker() is necessary, otherwise Encode() generates partial groups
-			nalus, pts, err := t.decoder.DecodeUntilMarker(pkt)
+			au, pts, err := t.decoder.DecodeUntilMarker(pkt)
 			if err != nil {
 				if err == rtph265.ErrNonStartingPacketAndNoPrevious || err == rtph265.ErrMorePacketsNeeded {
 					return nil
@@ -183,10 +253,9 @@ func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error {
 				return err
 			}
 
-			tdata.nalus = nalus
+			tdata.au = au
 			tdata.pts = pts
-
-			tdata.nalus = t.remuxNALUs(tdata.nalus)
+			tdata.au = t.remuxAccessUnit(tdata.au)
 		}
 
 		// route packet as is
@@ -194,11 +263,11 @@ func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error {
 			return nil
 		}
 	} else {
-		t.updateTrackParametersFromNALUs(tdata.nalus)
-		tdata.nalus = t.remuxNALUs(tdata.nalus)
+		t.updateTrackParametersFromNALUs(tdata.au)
+		tdata.au = t.remuxAccessUnit(tdata.au)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.nalus, tdata.pts)
+	pkts, err := t.encoder.Encode(tdata.au, tdata.pts)
 	if err != nil {
 		return err
 	}

internal/core/hls_muxer.go

@@ -243,38 +243,51 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 		m.path.readerRemove(pathReaderRemoveReq{author: m})
 	}()
 
-	var videoFormat *format.H264
-	videoMedia := res.stream.medias().FindFormat(&videoFormat)
+	m.ringBuffer, _ = ringbuffer.New(uint64(m.readBufferCount))
 
-	var audioFormat *format.MPEG4Audio
-	audioMedia := res.stream.medias().FindFormat(&audioFormat)
+	var medias media.Medias
+
+	var videoFormat format.Format
+	var videoMedia *media.Media
+
+	var videoFormatH265 *format.H265
+	videoMedia = res.stream.medias().FindFormat(&videoFormatH265)
+
+	if videoFormatH265 != nil {
+		videoFormat = videoFormatH265
+		medias = append(medias, videoMedia)
 
-	if videoFormat == nil && audioFormat == nil {
-		return fmt.Errorf("the stream doesn't contain an H264 track or an AAC track")
+		videoStartPTSFilled := false
+		var videoStartPTS time.Duration
+
+		res.stream.readerAdd(m, videoMedia, videoFormat, func(dat data) {
+			m.ringBuffer.Push(func() error {
+				tdata := dat.(*dataH265)
+
+				if tdata.au == nil {
+					return nil
 				}
 
-	var err error
-	m.muxer, err = hls.NewMuxer(
-		hls.MuxerVariant(m.hlsVariant),
-		m.hlsSegmentCount,
-		time.Duration(m.hlsSegmentDuration),
-		time.Duration(m.hlsPartDuration),
-		uint64(m.hlsSegmentMaxSize),
-		videoFormat,
-		audioFormat,
-	)
+				if !videoStartPTSFilled {
+					videoStartPTSFilled = true
+					videoStartPTS = tdata.pts
+				}
+				pts := tdata.pts - videoStartPTS
+
+				err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
 				if err != nil {
 					return fmt.Errorf("muxer error: %v", err)
 				}
-	defer m.muxer.Close()
 
-	innerReady <- struct{}{}
-
-	m.ringBuffer, _ = ringbuffer.New(uint64(m.readBufferCount))
-
-	var medias media.Medias
+				return nil
+			})
+		})
+	} else {
+		var videoFormatH264 *format.H264
+		videoMedia = res.stream.medias().FindFormat(&videoFormatH264)
 
-	if videoMedia != nil {
+		if videoFormatH264 != nil {
+			videoFormat = videoFormatH264
 			medias = append(medias, videoMedia)
 
 			videoStartPTSFilled := false
@@ -284,7 +297,7 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 				m.ringBuffer.Push(func() error {
 					tdata := dat.(*dataH264)
 
-				if tdata.nalus == nil {
+					if tdata.au == nil {
 						return nil
 					}
 
@@ -294,7 +307,7 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 					}
 					pts := tdata.pts - videoStartPTS
 
-				err := m.muxer.WriteH264(tdata.ntp, pts, tdata.nalus)
+					err := m.muxer.WriteH26x(tdata.ntp, pts, tdata.au)
 					if err != nil {
 						return fmt.Errorf("muxer error: %v", err)
 					}
@@ -303,8 +316,12 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 				})
 			})
 		}
+	}
+
+	var audioFormat *format.MPEG4Audio
+	audioMedia := res.stream.medias().FindFormat(&audioFormat)
 
-	if audioMedia != nil {
+	if audioFormat != nil {
 		medias = append(medias, audioMedia)
 
 		audioStartPTSFilled := false
@@ -342,6 +359,27 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
 
 	defer res.stream.readerRemove(m)
 
+	if medias == nil {
+		return fmt.Errorf("the stream doesn't contain a supported video or audio track")
+	}
+
+	var err error
+	m.muxer, err = hls.NewMuxer(
+		hls.MuxerVariant(m.hlsVariant),
+		m.hlsSegmentCount,
+		time.Duration(m.hlsSegmentDuration),
+		time.Duration(m.hlsPartDuration),
+		uint64(m.hlsSegmentMaxSize),
+		videoFormat,
+		audioFormat,
+	)
+	if err != nil {
+		return fmt.Errorf("muxer error: %v", err)
+	}
+	defer m.muxer.Close()
+
+	innerReady <- struct{}{}
+
 	m.log(logger.Info, "is converting into HLS, %s",
 		sourceMediaInfo(medias))
 

internal/core/hls_source.go

@@ -84,10 +84,10 @@ func (s *hlsSource) run(ctx context.Context) error {
 		return nil
 	}
 
-	onVideoData := func(pts time.Duration, nalus [][]byte) {
+	onVideoData := func(pts time.Duration, au [][]byte) {
 		err := stream.writeData(videoMedia, videoMedia.Formats[0], &dataH264{
 			pts: pts,
-			nalus: nalus,
+			au:  au,
 			ntp: time.Now(),
 		})
 		if err != nil {

internal/core/rpicamera_source.go

@@ -48,7 +48,7 @@ func (s *rpiCameraSource) run(ctx context.Context) error {
 	medias := media.Medias{medi}
 	var stream *stream
 
-	onData := func(dts time.Duration, nalus [][]byte) {
+	onData := func(dts time.Duration, au [][]byte) {
 		if stream == nil {
 			res := s.parent.sourceStaticImplSetReady(pathSourceStaticSetReadyReq{
 				medias:             medias,
@@ -64,7 +64,7 @@ func (s *rpiCameraSource) run(ctx context.Context) error {
 
 		err := stream.writeData(medi, medi.Formats[0], &dataH264{
 			pts: dts,
-			nalus: nalus,
+			au:  au,
 			ntp: time.Now(),
 		})
 		if err != nil {

internal/core/rtmp_conn.go

@@ -281,7 +281,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
 			ringBuffer.Push(func() error {
 				tdata := dat.(*dataH264)
 
-				if tdata.nalus == nil {
+				if tdata.au == nil {
 					return nil
 				}
 
@@ -294,7 +294,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
 				idrPresent := false
 				nonIDRPresent := false
 
-				for _, nalu := range tdata.nalus {
+				for _, nalu := range tdata.au {
 					typ := h264.NALUType(nalu[0] & 0x1F)
 					switch typ {
 					case h264.NALUTypeIDR:
@@ -317,7 +317,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
 					videoDTSExtractor = h264.NewDTSExtractor()
 
 					var err error
-					dts, err = videoDTSExtractor.Extract(tdata.nalus, pts)
+					dts, err = videoDTSExtractor.Extract(tdata.au, pts)
 					if err != nil {
 						return err
 					}
@@ -331,7 +331,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
 					}
 
 					var err error
-					dts, err = videoDTSExtractor.Extract(tdata.nalus, pts)
+					dts, err = videoDTSExtractor.Extract(tdata.au, pts)
 					if err != nil {
 						return err
 					}
@@ -340,7 +340,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
 					pts -= videoStartDTS
 				}
 
-				avcc, err := h264.AVCCMarshal(tdata.nalus)
+				avcc, err := h264.AVCCMarshal(tdata.au)
 				if err != nil {
 					return err
 				}
@@ -538,10 +538,10 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
 	var onVideoData func(time.Duration, [][]byte)
 
 	if _, ok := videoFormat.(*format.H264); ok {
-		onVideoData = func(pts time.Duration, nalus [][]byte) {
+		onVideoData = func(pts time.Duration, au [][]byte) {
 			err = rres.stream.writeData(videoMedia, videoFormat, &dataH264{
 				pts: pts,
-				nalus: nalus,
+				au:  au,
 				ntp: time.Now(),
 			})
 			if err != nil {
@@ -549,10 +549,10 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
 			}
 		}
 	} else {
-		onVideoData = func(pts time.Duration, nalus [][]byte) {
+		onVideoData = func(pts time.Duration, au [][]byte) {
 			err = rres.stream.writeData(videoMedia, videoFormat, &dataH265{
 				pts: pts,
-				nalus: nalus,
+				au:  au,
 				ntp: time.Now(),
 			})
 			if err != nil {
@@ -577,14 +577,14 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
 					return fmt.Errorf("unable to parse H264 config: %v", err)
 				}
 
-				nalus := [][]byte{
+				au := [][]byte{
 					conf.SPS,
 					conf.PPS,
 				}
 
 				err := rres.stream.writeData(videoMedia, videoFormat, &dataH264{
 					pts: tmsg.DTS + tmsg.PTSDelta,
-					nalus: nalus,
+					au:  au,
 					ntp: time.Now(),
 				})
 				if err != nil {

internal/core/rtmp_source.go

@@ -176,14 +176,14 @@ func (s *rtmpSource) run(ctx context.Context) error {
 							return fmt.Errorf("received an H264 packet, but track is not set up")
 						}
 
-						nalus, err := h264.AVCCUnmarshal(tmsg.Payload)
+						au, err := h264.AVCCUnmarshal(tmsg.Payload)
 						if err != nil {
 							return fmt.Errorf("unable to decode AVCC: %v", err)
 						}
 
 						err = res.stream.writeData(videoMedia, videoFormat, &dataH264{
 							pts: tmsg.DTS + tmsg.PTSDelta,
-							nalus: nalus,
+							au:  au,
 							ntp: time.Now(),
 						})
 						if err != nil {

internal/core/webrtc_conn.go

@@ -519,12 +519,12 @@ func (c *webRTCConn) allocateTracks(medias media.Medias) ([]*webRTCTrack, error)
 				cb: func(dat data, ctx context.Context, writeError chan error) {
 					tdata := dat.(*dataH264)
 
-					if tdata.nalus == nil {
+					if tdata.au == nil {
 						return
 					}
 
 					if !firstNALUReceived {
-						if !h264.IDRPresent(tdata.nalus) {
+						if !h264.IDRPresent(tdata.au) {
 							return
 						}
 
@@ -541,7 +541,7 @@ func (c *webRTCConn) allocateTracks(medias media.Medias) ([]*webRTCTrack, error)
 						lastPTS = tdata.pts
 					}
 
-					packets, err := encoder.Encode(tdata.nalus, tdata.pts)
+					packets, err := encoder.Encode(tdata.au, tdata.pts)
 					if err != nil {
 						return
 					}

internal/hls/fmp4/init_track.go

@@ -3,6 +3,7 @@ package fmp4
 import (
 	gomp4 "github.com/abema/go-mp4"
 	"github.com/aler9/gortsplib/v2/pkg/codecs/h264"
+	"github.com/aler9/gortsplib/v2/pkg/codecs/h265"
 	"github.com/aler9/gortsplib/v2/pkg/format"
 )
 
@@ -46,24 +47,30 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 		return err
 	}
 
-	var sps []byte
-	var pps []byte
-	var spsp h264.SPS
+	var h264SPS []byte
+	var h264PPS []byte
+	var h264SPSP h264.SPS
+
+	var h265VPS []byte
+	var h265SPS []byte
+	var h265PPS []byte
+	var h265SPSP h265.SPS
+
 	var width int
 	var height int
 
 	switch ttrack := track.Format.(type) {
 	case *format.H264:
-		sps = ttrack.SafeSPS()
-		pps = ttrack.SafePPS()
+		h264SPS = ttrack.SafeSPS()
+		h264PPS = ttrack.SafePPS()
 
-		err = spsp.Unmarshal(sps)
+		err = h264SPSP.Unmarshal(h264SPS)
 		if err != nil {
 			return err
 		}
 
-		width = spsp.Width()
-		height = spsp.Height()
+		width = h264SPSP.Width()
+		height = h264SPSP.Height()
 
 		_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
 			FullBox: gomp4.FullBox{
@@ -72,7 +79,33 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 			TrackID: uint32(track.ID),
 			Width:   uint32(width * 65536),
 			Height:  uint32(height * 65536),
-			Matrix:  [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000},
+			Matrix:  [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
+		})
+		if err != nil {
+			return err
+		}
+
+	case *format.H265:
+		h265VPS = ttrack.SafeVPS()
+		h265SPS = ttrack.SafeSPS()
+		h265PPS = ttrack.SafePPS()
+
+		err = h265SPSP.Unmarshal(h265SPS)
+		if err != nil {
+			return err
+		}
+
+		width = h265SPSP.Width()
+		height = h265SPSP.Height()
+
+		_, err = w.WriteBox(&gomp4.Tkhd{ // <tkhd/>
+			FullBox: gomp4.FullBox{
+				Flags: [3]byte{0, 0, 3},
+			},
+			TrackID: uint32(track.ID),
+			Width:   uint32(width * 65536),
+			Height:  uint32(height * 65536),
+			Matrix:  [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
 		})
 		if err != nil {
 			return err
@@ -86,7 +119,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 			TrackID:        uint32(track.ID),
 			AlternateGroup: 1,
 			Volume:         256,
-			Matrix:         [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000},
+			Matrix:         [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000},
 		})
 		if err != nil {
 			return err
@@ -107,7 +140,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 	}
 
 	switch track.Format.(type) {
-	case *format.H264:
+	case *format.H264, *format.H265:
 		_, err = w.WriteBox(&gomp4.Hdlr{ // <hdlr/>
 			HandlerType: [4]byte{'v', 'i', 'd', 'e'},
 			Name:        "VideoHandler",
@@ -132,7 +165,7 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 	}
 
 	switch track.Format.(type) {
-	case *format.H264:
+	case *format.H264, *format.H265:
 		_, err = w.WriteBox(&gomp4.Vmhd{ // <vmhd/>
 			FullBox: gomp4.FullBox{
 				Flags: [3]byte{0, 0, 1},
@@ -219,22 +252,22 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 				Type: gomp4.BoxTypeAvcC(),
 			},
 			ConfigurationVersion:       1,
-			Profile:                    spsp.ProfileIdc,
-			ProfileCompatibility:       sps[2],
-			Level:                      spsp.LevelIdc,
+			Profile:                    h264SPSP.ProfileIdc,
+			ProfileCompatibility:       h264SPS[2],
+			Level:                      h264SPSP.LevelIdc,
 			LengthSizeMinusOne:         3,
 			NumOfSequenceParameterSets: 1,
 			SequenceParameterSets: []gomp4.AVCParameterSet{
 				{
-					Length:  uint16(len(sps)),
-					NALUnit: sps,
+					Length:  uint16(len(h264SPS)),
+					NALUnit: h264SPS,
 				},
 			},
 			NumOfPictureParameterSets: 1,
 			PictureParameterSets: []gomp4.AVCParameterSet{
 				{
-					Length:  uint16(len(pps)),
-					NALUnit: pps,
+					Length:  uint16(len(h264PPS)),
+					NALUnit: h264PPS,
 				},
 			},
 		})
@@ -255,6 +288,90 @@ func (track *InitTrack) marshal(w *mp4Writer) error {
 			return err
 		}
 
+	case *format.H265:
+		_, err = w.writeBoxStart(&gomp4.VisualSampleEntry{ // <hev1>
+			SampleEntry: gomp4.SampleEntry{
+				AnyTypeBox: gomp4.AnyTypeBox{
+					Type: gomp4.BoxTypeHev1(),
+				},
+				DataReferenceIndex: 1,
+			},
+			Width:           uint16(width),
+			Height:          uint16(height),
+			Horizresolution: 4718592,
+			Vertresolution:  4718592,
+			FrameCount:      1,
+			Depth:           24,
+			PreDefined3:     -1,
+		})
+		if err != nil {
+			return err
+		}
+
+		_, err = w.WriteBox(&gomp4.HvcC{ // <hvcC/>
+			ConfigurationVersion:        1,
+			GeneralProfileIdc:           h265SPSP.ProfileTierLevel.GeneralProfileIdc,
+			GeneralProfileCompatibility: h265SPSP.ProfileTierLevel.GeneralProfileCompatibilityFlag,
+			GeneralConstraintIndicator: [6]uint8{
+				h265SPS[7], h265SPS[8], h265SPS[9],
+				h265SPS[10], h265SPS[11], h265SPS[12],
+			},
+			GeneralLevelIdc: h265SPSP.ProfileTierLevel.GeneralLevelIdc,
+			// MinSpatialSegmentationIdc
+			// ParallelismType
+			ChromaFormatIdc:      uint8(h265SPSP.ChromaFormatIdc),
+			BitDepthLumaMinus8:   uint8(h265SPSP.BitDepthLumaMinus8),
+			BitDepthChromaMinus8: uint8(h265SPSP.BitDepthChromaMinus8),
+			// AvgFrameRate
+			// ConstantFrameRate
+			NumTemporalLayers: 1,
+			// TemporalIdNested
+			LengthSizeMinusOne: 3,
+			NumOfNaluArrays:    3,
+			NaluArrays: []gomp4.HEVCNaluArray{
+				{
+					NaluType: byte(h265.NALUType_VPS_NUT),
+					NumNalus: 1,
+					Nalus: []gomp4.HEVCNalu{{
+						Length:  uint16(len(h265VPS)),
+						NALUnit: h265VPS,
+					}},
+				},
+				{
+					NaluType: byte(h265.NALUType_SPS_NUT),
+					NumNalus: 1,
+					Nalus: []gomp4.HEVCNalu{{
+						Length:  uint16(len(h265SPS)),
+						NALUnit: h265SPS,
+					}},
+				},
+				{
+					NaluType: byte(h265.NALUType_PPS_NUT),
+					NumNalus: 1,
+					Nalus: []gomp4.HEVCNalu{{
+						Length:  uint16(len(h265PPS)),
+						NALUnit: h265PPS,
+					}},
+				},
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		_, err = w.WriteBox(&gomp4.Btrt{ // <btrt/>
+			MaxBitrate: 1000000,
+			AvgBitrate: 1000000,
+		})
+		if err != nil {
+			return err
+		}
+
+		err = w.writeBoxEnd() // </hev1>
+		if err != nil {
+			return err
+		}
+
 	case *format.MPEG4Audio:
 		_, err = w.writeBoxStart(&gomp4.AudioSampleEntry{ // <mp4a>
 			SampleEntry: gomp4.SampleEntry{

internal/hls/mpegts/writer.go

@@ -86,7 +86,7 @@ func (w *Writer) GenerateSegment() []byte {
 	return ret
 }
 
-// WriteH264 writes a group of H264 NALUs.
+// WriteH264 writes a H264 access unit.
 func (w *Writer) WriteH264(
 	pcr time.Duration,
 	dts time.Duration,

internal/hls/muxer.go

@@ -28,20 +28,24 @@ func NewMuxer(
 	segmentDuration time.Duration,
 	partDuration time.Duration,
 	segmentMaxSize uint64,
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
 ) (*Muxer, error) {
 	m := &Muxer{}
 
 	switch variant {
 	case MuxerVariantMPEGTS:
-		m.variant = newMuxerVariantMPEGTS(
+		var err error
+		m.variant, err = newMuxerVariantMPEGTS(
 			segmentCount,
 			segmentDuration,
 			segmentMaxSize,
 			videoTrack,
 			audioTrack,
 		)
+		if err != nil {
+			return nil, err
+		}
 
 	case MuxerVariantFMP4:
 		m.variant = newMuxerVariantFMP4(
@@ -76,12 +80,12 @@ func (m *Muxer) Close() {
 	m.variant.close()
 }
 
-// WriteH264 writes H264 NALUs, grouped by timestamp.
-func (m *Muxer) WriteH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
-	return m.variant.writeH264(ntp, pts, nalus)
+// WriteH26x writes an H264 or an H265 access unit.
+func (m *Muxer) WriteH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
+	return m.variant.writeH26x(ntp, pts, au)
 }
 
-// WriteAAC writes AAC AUs, grouped by timestamp.
+// WriteAAC writes an AAC access unit.
 func (m *Muxer) WriteAAC(ntp time.Time, pts time.Duration, au []byte) error {
 	return m.variant.writeAAC(ntp, pts, au)
 }

internal/hls/muxer_primary_playlist.go

@@ -8,18 +8,43 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/aler9/gortsplib/v2/pkg/codecs/h265"
 	"github.com/aler9/gortsplib/v2/pkg/format"
 )
 
+func codecParameters(track format.Format) string {
+	switch ttrack := track.(type) {
+	case *format.H264:
+		sps := ttrack.SafeSPS()
+		if len(sps) >= 4 {
+			return "avc1." + hex.EncodeToString(sps[1:4])
+		}
+
+	case *format.H265:
+		var sps h265.SPS
+		err := sps.Unmarshal(ttrack.SafeSPS())
+		if err == nil {
+			return "hvc1." + strconv.FormatInt(int64(sps.ProfileTierLevel.GeneralProfileIdc), 10) +
+				".4.L" + strconv.FormatInt(int64(sps.ProfileTierLevel.GeneralLevelIdc), 10) + ".B0"
+		}
+
+	case *format.MPEG4Audio:
+		// https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
+		return "mp4a.40." + strconv.FormatInt(int64(ttrack.Config.Type), 10)
+	}
+
+	return ""
+}
+
 type muxerPrimaryPlaylist struct {
 	fmp4       bool
-	videoTrack *format.H264
+	videoTrack format.Format
 	audioTrack *format.MPEG4Audio
 }
 
 func newMuxerPrimaryPlaylist(
 	fmp4 bool,
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
 ) *muxerPrimaryPlaylist {
 	return &muxerPrimaryPlaylist{
@@ -39,15 +64,10 @@ func (p *muxerPrimaryPlaylist) file() *MuxerFileResponse {
 			var codecs []string
 
 			if p.videoTrack != nil {
-				sps := p.videoTrack.SafeSPS()
-				if len(sps) >= 4 {
-					codecs = append(codecs, "avc1."+hex.EncodeToString(sps[1:4]))
+				codecs = append(codecs, codecParameters(p.videoTrack))
 			}
-			}
-
-			// https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
 			if p.audioTrack != nil {
-				codecs = append(codecs, "mp4a.40."+strconv.FormatInt(int64(p.audioTrack.Config.Type), 10))
+				codecs = append(codecs, codecParameters(p.audioTrack))
 			}
 
 			var version int

internal/hls/muxer_test.go

@@ -57,17 +57,17 @@ func TestMuxerVideoAudio(t *testing.T) {
 			require.NoError(t, err)
 			defer m.Close()
 
-			// group without IDR
+			// access unit without IDR
 			d := 1 * time.Second
-			err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
 				{0x06},
 				{0x07},
 			})
 			require.NoError(t, err)
 
-			// group with IDR
+			// access unit with IDR
 			d = 2 * time.Second
-			err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
 				testSPS, // SPS
 				{8},     // PPS
 				{5},     // IDR
@@ -86,9 +86,9 @@ func TestMuxerVideoAudio(t *testing.T) {
 			})
 			require.NoError(t, err)
 
-			// group without IDR
+			// access unit without IDR
 			d = 4 * time.Second
-			err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
 				{1}, // non-IDR
 			})
 			require.NoError(t, err)
@@ -99,16 +99,16 @@ func TestMuxerVideoAudio(t *testing.T) {
 			})
 			require.NoError(t, err)
 
-			// group with IDR
+			// access unit with IDR
 			d = 6 * time.Second
-			err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
 				{5}, // IDR
 			})
 			require.NoError(t, err)
 
-			// group with IDR
+			// access unit with IDR
 			d = 7 * time.Second
-			err = m.WriteH264(testTime.Add(d-1*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-1*time.Second), d, [][]byte{
 				{5}, // IDR
 			})
 			require.NoError(t, err)
@@ -203,25 +203,25 @@ func TestMuxerVideoOnly(t *testing.T) {
 			require.NoError(t, err)
 			defer m.Close()
 
-			// group with IDR
+			// access unit with IDR
 			d := 2 * time.Second
-			err = m.WriteH264(testTime.Add(d-2*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-2*time.Second), d, [][]byte{
 				testSPS, // SPS
 				{8},     // PPS
 				{5},     // IDR
 			})
 			require.NoError(t, err)
 
-			// group with IDR
+			// access unit with IDR
 			d = 6 * time.Second
-			err = m.WriteH264(testTime.Add(d-2*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-2*time.Second), d, [][]byte{
 				{5}, // IDR
 			})
 			require.NoError(t, err)
 
-			// group with IDR
+			// access unit with IDR
 			d = 7 * time.Second
-			err = m.WriteH264(testTime.Add(d-2*time.Second), d, [][]byte{
+			err = m.WriteH26x(testTime.Add(d-2*time.Second), d, [][]byte{
 				{5}, // IDR
 			})
 			require.NoError(t, err)
@@ -415,8 +415,8 @@ func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) {
 	m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil)
 	require.NoError(t, err)
 
-	// group with IDR
-	err = m.WriteH264(testTime, 2*time.Second, [][]byte{
+	// access unit with IDR
+	err = m.WriteH26x(testTime, 2*time.Second, [][]byte{
 		testSPS, // SPS
 		{8},     // PPS
 		{5},     // IDR
@@ -441,7 +441,7 @@ func TestMuxerMaxSegmentSize(t *testing.T) {
 	require.NoError(t, err)
 	defer m.Close()
 
-	err = m.WriteH264(testTime, 2*time.Second, [][]byte{
+	err = m.WriteH26x(testTime, 2*time.Second, [][]byte{
 		testSPS,
 		{5}, // IDR
 	})
@@ -460,14 +460,14 @@ func TestMuxerDoubleRead(t *testing.T) {
 	require.NoError(t, err)
 	defer m.Close()
 
-	err = m.WriteH264(testTime, 0, [][]byte{
+	err = m.WriteH26x(testTime, 0, [][]byte{
 		testSPS,
 		{5}, // IDR
 		{1},
 	})
 	require.NoError(t, err)
 
-	err = m.WriteH264(testTime, 2*time.Second, [][]byte{
+	err = m.WriteH26x(testTime, 2*time.Second, [][]byte{
 		{5}, // IDR
 		{2},
 	})

internal/hls/muxer_variant.go

@@ -16,7 +16,7 @@ const (
 
 type muxerVariant interface {
 	close()
-	writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error
-	writeAAC(ntp time.Time, pts time.Duration, au []byte) error
+	writeH26x(time.Time, time.Duration, [][]byte) error
+	writeAAC(time.Time, time.Duration, []byte) error
 	file(name string, msn string, part string, skip string) *MuxerFileResponse
 }

internal/hls/muxer_variant_fmp4.go

@@ -11,15 +11,47 @@ import (
 	"github.com/aler9/rtsp-simple-server/internal/hls/fmp4"
 )
 
+func extractVideoParams(track format.Format) [][]byte {
+	switch ttrack := track.(type) {
+	case *format.H264:
+		params := make([][]byte, 2)
+		params[0] = ttrack.SafeSPS()
+		params[1] = ttrack.SafePPS()
+		return params
+
+	case *format.H265:
+		params := make([][]byte, 3)
+		params[0] = ttrack.SafeVPS()
+		params[1] = ttrack.SafeSPS()
+		params[2] = ttrack.SafePPS()
+		return params
+
+	default:
+		return nil
+	}
+}
+
+func videoParamsEqual(p1 [][]byte, p2 [][]byte) bool {
+	if len(p1) != len(p2) {
+		return true
+	}
+
+	for i, p := range p1 {
+		if !bytes.Equal(p2[i], p) {
+			return false
+		}
+	}
+	return true
+}
+
 type muxerVariantFMP4 struct {
 	playlist   *muxerVariantFMP4Playlist
 	segmenter  *muxerVariantFMP4Segmenter
-	videoTrack *format.H264
+	videoTrack format.Format
 	audioTrack *format.MPEG4Audio
 
 	mutex           sync.Mutex
-	videoLastSPS []byte
-	videoLastPPS []byte
+	lastVideoParams [][]byte
 	initContent     []byte
 }
 
@@ -29,7 +61,7 @@ func newMuxerVariantFMP4(
 	segmentDuration time.Duration,
 	partDuration time.Duration,
 	segmentMaxSize uint64,
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
 ) *muxerVariantFMP4 {
 	v := &muxerVariantFMP4{
@@ -63,28 +95,34 @@ func (v *muxerVariantFMP4) close() {
 	v.playlist.close()
 }
 
-func (v *muxerVariantFMP4) writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
-	return v.segmenter.writeH264(ntp, pts, nalus)
+func (v *muxerVariantFMP4) writeH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
+	return v.segmenter.writeH26x(ntp, pts, au)
 }
 
 func (v *muxerVariantFMP4) writeAAC(ntp time.Time, pts time.Duration, au []byte) error {
 	return v.segmenter.writeAAC(ntp, pts, au)
 }
 
+func (v *muxerVariantFMP4) mustRegenerateInit() bool {
+	if v.videoTrack == nil {
+		return false
+	}
+
+	videoParams := extractVideoParams(v.videoTrack)
+	if !videoParamsEqual(videoParams, v.lastVideoParams) {
+		v.lastVideoParams = videoParams
+		return true
+	}
+
+	return false
+}
+
 func (v *muxerVariantFMP4) file(name string, msn string, part string, skip string) *MuxerFileResponse {
 	if name == "init.mp4" {
 		v.mutex.Lock()
 		defer v.mutex.Unlock()
 
-		var sps []byte
-		var pps []byte
-		if v.videoTrack != nil {
-			sps = v.videoTrack.SafeSPS()
-			pps = v.videoTrack.SafePPS()
-		}
-
-		if v.initContent == nil ||
-			(v.videoTrack != nil && (!bytes.Equal(v.videoLastSPS, sps) || !bytes.Equal(v.videoLastPPS, pps))) {
+		if v.initContent == nil || v.mustRegenerateInit() {
 			init := fmp4.Init{}
 			trackID := 1
 
@@ -105,14 +143,11 @@ func (v *muxerVariantFMP4) file(name string, msn string, part string, skip strin
 				})
 			}
 
-			initContent, err := init.Marshal()
+			var err error
+			v.initContent, err = init.Marshal()
 			if err != nil {
 				return &MuxerFileResponse{Status: http.StatusInternalServerError}
 			}
-
-			v.videoLastSPS = sps
-			v.videoLastPPS = pps
-			v.initContent = initContent
 		}
 
 		return &MuxerFileResponse{

internal/hls/muxer_variant_fmp4_part.go

@@ -17,7 +17,7 @@ func fmp4PartName(id uint64) string {
 }
 
 type muxerVariantFMP4Part struct {
-	videoTrack *format.H264
+	videoTrack format.Format
 	audioTrack *format.MPEG4Audio
 	id         uint64
 
@@ -33,7 +33,7 @@ type muxerVariantFMP4Part struct {
 }
 
 func newMuxerVariantFMP4Part(
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
 	id uint64,
 ) *muxerVariantFMP4Part {

internal/hls/muxer_variant_fmp4_playlist.go

@@ -70,7 +70,7 @@ func partTargetDuration(
 type muxerVariantFMP4Playlist struct {
 	lowLatency   bool
 	segmentCount int
-	videoTrack   *format.H264
+	videoTrack   format.Format
 	audioTrack   *format.MPEG4Audio
 
 	mutex              sync.Mutex
@@ -89,7 +89,7 @@ type muxerVariantFMP4Playlist struct {
 func newMuxerVariantFMP4Playlist(
 	lowLatency bool,
 	segmentCount int,
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
 ) *muxerVariantFMP4Playlist {
 	p := &muxerVariantFMP4Playlist{

internal/hls/muxer_variant_fmp4_segment.go

@@ -45,7 +45,7 @@ type muxerVariantFMP4Segment struct {
 	startTime       time.Time
 	startDTS        time.Duration
 	segmentMaxSize  uint64
-	videoTrack      *format.H264
+	videoTrack      format.Format
 	audioTrack      *format.MPEG4Audio
 	genPartID       func() uint64
 	onPartFinalized func(*muxerVariantFMP4Part)
@@ -63,7 +63,7 @@ func newMuxerVariantFMP4Segment(
 	startTime time.Time,
 	startDTS time.Duration,
 	segmentMaxSize uint64,
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
 	genPartID func() uint64,
 	onPartFinalized func(*muxerVariantFMP4Part),

internal/hls/muxer_variant_fmp4_segmenter.go

@@ -1,10 +1,11 @@
 package hls
 
 import (
-	"bytes"
+	"fmt"
 	"time"
 
 	"github.com/aler9/gortsplib/v2/pkg/codecs/h264"
+	"github.com/aler9/gortsplib/v2/pkg/codecs/h265"
 	"github.com/aler9/gortsplib/v2/pkg/format"
 
 	"github.com/aler9/rtsp-simple-server/internal/hls/fmp4"
@@ -45,6 +46,21 @@ func findCompatiblePartDuration(
 	return i
 }
 
+type dtsExtractor interface {
+	Extract([][]byte, time.Duration) (time.Duration, error)
+}
+
+func allocateDTSExtractor(track format.Format) dtsExtractor {
+	switch track.(type) {
+	case *format.H264:
+		return h264.NewDTSExtractor()
+
+	case *format.H265:
+		return h265.NewDTSExtractor()
+	}
+	return nil
+}
+
 type augmentedVideoSample struct {
 	fmp4.PartSample
 	dts time.Duration
@@ -62,15 +78,15 @@ type muxerVariantFMP4Segmenter struct {
 	segmentDuration    time.Duration
 	partDuration       time.Duration
 	segmentMaxSize     uint64
-	videoTrack         *format.H264
+	videoTrack         format.Format
 	audioTrack         *format.MPEG4Audio
 	onSegmentFinalized func(*muxerVariantFMP4Segment)
 	onPartFinalized    func(*muxerVariantFMP4Part)
 
 	startDTS                       time.Duration
-	videoFirstIDRReceived bool
-	videoDTSExtractor     *h264.DTSExtractor
-	videoSPS              []byte
+	videoFirstRandomAccessReceived bool
+	videoDTSExtractor              dtsExtractor
+	lastVideoParams                [][]byte
 	currentSegment                 *muxerVariantFMP4Segment
 	nextSegmentID                  uint64
 	nextPartID                     uint64
@@ -87,7 +103,7 @@ func newMuxerVariantFMP4Segmenter(
 	segmentDuration time.Duration,
 	partDuration time.Duration,
 	segmentMaxSize uint64,
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
 	onSegmentFinalized func(*muxerVariantFMP4Segment),
 	onPartFinalized func(*muxerVariantFMP4Part),
@@ -140,50 +156,65 @@ func (m *muxerVariantFMP4Segmenter) adjustPartDuration(du time.Duration) {
 	}
 }
 
-func (m *muxerVariantFMP4Segmenter) writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
-	idrPresent := false
+func (m *muxerVariantFMP4Segmenter) writeH26x(ntp time.Time, pts time.Duration, au [][]byte) error {
+	randomAccessPresent := false
+
+	switch m.videoTrack.(type) {
+	case *format.H264:
 		nonIDRPresent := false
 
-	for _, nalu := range nalus {
+		for _, nalu := range au {
 			typ := h264.NALUType(nalu[0] & 0x1F)
+
 			switch typ {
 			case h264.NALUTypeIDR:
-			idrPresent = true
+				randomAccessPresent = true
 
 			case h264.NALUTypeNonIDR:
 				nonIDRPresent = true
 			}
 		}
 
-	if !idrPresent && !nonIDRPresent {
+		if !randomAccessPresent && !nonIDRPresent {
 			return nil
 		}
 
-	return m.writeH264Entry(ntp, pts, nalus, idrPresent)
+	case *format.H265:
+		for _, nalu := range au {
+			typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+			switch typ {
+			case h265.NALUType_IDR_W_RADL, h265.NALUType_IDR_N_LP, h265.NALUType_CRA_NUT:
+				randomAccessPresent = true
+			}
+		}
 	}
 
-func (m *muxerVariantFMP4Segmenter) writeH264Entry(
+	return m.writeH26xEntry(ntp, pts, au, randomAccessPresent)
+}
+
+func (m *muxerVariantFMP4Segmenter) writeH26xEntry(
 	ntp time.Time,
 	pts time.Duration,
-	nalus [][]byte,
-	idrPresent bool,
+	au [][]byte,
+	randomAccessPresent bool,
 ) error {
 	var dts time.Duration
 
-	if !m.videoFirstIDRReceived {
+	if !m.videoFirstRandomAccessReceived {
 		// skip sample silently until we find one with an IDR
-		if !idrPresent {
+		if !randomAccessPresent {
 			return nil
 		}
 
-		m.videoFirstIDRReceived = true
-		m.videoDTSExtractor = h264.NewDTSExtractor()
-		m.videoSPS = m.videoTrack.SafeSPS()
+		m.videoFirstRandomAccessReceived = true
+		m.videoDTSExtractor = allocateDTSExtractor(m.videoTrack)
+		m.lastVideoParams = extractVideoParams(m.videoTrack)
 
 		var err error
-		dts, err = m.videoDTSExtractor.Extract(nalus, pts)
+		dts, err = m.videoDTSExtractor.Extract(au, pts)
 		if err != nil {
-			return err
+			return fmt.Errorf("unable to extract DTS: %v", err)
 		}
 
 		m.startDTS = dts
@@ -191,16 +222,16 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
 		pts -= m.startDTS
 	} else {
 		var err error
-		dts, err = m.videoDTSExtractor.Extract(nalus, pts)
+		dts, err = m.videoDTSExtractor.Extract(au, pts)
 		if err != nil {
-			return err
+			return fmt.Errorf("unable to extract DTS: %v", err)
 		}
 
 		dts -= m.startDTS
 		pts -= m.startDTS
 	}
 
-	avcc, err := h264.AVCCMarshal(nalus)
+	avcc, err := h264.AVCCMarshal(au)
 	if err != nil {
 		return err
 	}
@@ -208,7 +239,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
 	sample := &augmentedVideoSample{
 		PartSample: fmp4.PartSample{
 			PTSOffset:       int32(durationGoToMp4(pts-dts, 90000)),
-			IsNonSyncSample: !idrPresent,
+			IsNonSyncSample: !randomAccessPresent,
 			Payload:         avcc,
 		},
 		dts: dts,
@@ -247,12 +278,12 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
 	}
 
 	// switch segment
-	if idrPresent {
-		sps := m.videoTrack.SafeSPS()
-		spsChanged := !bytes.Equal(m.videoSPS, sps)
+	if randomAccessPresent {
+		videoParams := extractVideoParams(m.videoTrack)
+		paramsChanged := !videoParamsEqual(m.lastVideoParams, videoParams)
 
 		if (m.nextVideoSample.dts-m.currentSegment.startDTS) >= m.segmentDuration ||
-			spsChanged {
+			paramsChanged {
 			err := m.currentSegment.finalize(m.nextVideoSample.dts)
 			if err != nil {
 				return err
@@ -273,10 +304,11 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
 				m.onPartFinalized,
 			)
 
-			// if SPS changed, reset adjusted part duration
-			if spsChanged {
-				m.videoSPS = sps
+			if paramsChanged {
+				m.lastVideoParams = videoParams
 				m.firstSegmentFinalized = false
+
+				// reset adjusted part duration
 				m.sampleDurations = make(map[time.Duration]struct{})
 			}
 		}
@@ -288,7 +320,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
 func (m *muxerVariantFMP4Segmenter) writeAAC(ntp time.Time, dts time.Duration, au []byte) error {
 	if m.videoTrack != nil {
 		// wait for the video track
-		if !m.videoFirstIDRReceived {
+		if !m.videoFirstRandomAccessReceived {
 			return nil
 		}
 

internal/hls/muxer_variant_mpegts.go

@@ -1,6 +1,7 @@
 package hls
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/aler9/gortsplib/v2/pkg/format"
@@ -15,9 +16,19 @@ func newMuxerVariantMPEGTS(
 	segmentCount int,
 	segmentDuration time.Duration,
 	segmentMaxSize uint64,
-	videoTrack *format.H264,
+	videoTrack format.Format,
 	audioTrack *format.MPEG4Audio,
-) *muxerVariantMPEGTS {
+) (*muxerVariantMPEGTS, error) {
+	var videoTrackH264 *format.H264
+	if videoTrack != nil {
+		var ok bool
+		videoTrackH264, ok = videoTrack.(*format.H264)
+		if !ok {
+			return nil, fmt.Errorf(
+				"the MPEG-TS variant of HLS doesn't support H265. Use the fMP4 or Low-Latency variants instead")
+		}
+	}
+
 	v := &muxerVariantMPEGTS{}
 
 	v.playlist = newMuxerVariantMPEGTSPlaylist(segmentCount)
@@ -25,21 +36,21 @@ func newMuxerVariantMPEGTS(
 	v.segmenter = newMuxerVariantMPEGTSSegmenter(
 		segmentDuration,
 		segmentMaxSize,
-		videoTrack,
+		videoTrackH264,
 		audioTrack,
 		func(seg *muxerVariantMPEGTSSegment) {
 			v.playlist.pushSegment(seg)
 		},
 	)
 
-	return v
+	return v, nil
 }
 
 func (v *muxerVariantMPEGTS) close() {
 	v.playlist.close()
 }
 
-func (v *muxerVariantMPEGTS) writeH264(ntp time.Time, pts time.Duration, nalus [][]byte) error {
+func (v *muxerVariantMPEGTS) writeH26x(ntp time.Time, pts time.Duration, nalus [][]byte) error {
 	return v.segmenter.writeH264(ntp, pts, nalus)
 }
 

internal/hls/muxer_variant_mpegts_segmenter.go

@@ -1,6 +1,7 @@
 package hls
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/aler9/gortsplib/v2/pkg/codecs/h264"
@@ -84,7 +85,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(ntp time.Time, pts time.Duration
 		var err error
 		dts, err = m.videoDTSExtractor.Extract(nalus, pts)
 		if err != nil {
-			return err
+			return fmt.Errorf("unable to extract DTS: %v", err)
 		}
 
 		m.startPCR = ntp
@@ -108,7 +109,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(ntp time.Time, pts time.Duration
 		var err error
 		dts, err = m.videoDTSExtractor.Extract(nalus, pts)
 		if err != nil {
-			return err
+			return fmt.Errorf("unable to extract DTS: %v", err)
 		}
 
 		dts -= m.startDTS