update gortsplib

go.mod

@@ -5,7 +5,7 @@ go 1.17
 require (
 	code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
 	github.com/abema/go-mp4 v0.7.2
-	github.com/aler9/gortsplib v0.0.0-20220602092441-d19c37025dc4
+	github.com/aler9/gortsplib v0.0.0-20220602183811-46253a74b083
 	github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gin-gonic/gin v1.7.2

go.sum

@@ -6,8 +6,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/aler9/gortsplib v0.0.0-20220602092441-d19c37025dc4 h1:0xhKBSxHasMGLUgdE/kTw5QmywTcil4oQwh0+MmK+6o=
-github.com/aler9/gortsplib v0.0.0-20220602092441-d19c37025dc4/go.mod h1:i1e4CEs42IrbidMUNTSNOKmeGPCOHVX9P3BvPxzyMtI=
+github.com/aler9/gortsplib v0.0.0-20220602183811-46253a74b083 h1:VmY5jw7jBl4PTX7J9UIrslRRco1ELt9oDteq9CmecqU=
+github.com/aler9/gortsplib v0.0.0-20220602183811-46253a74b083/go.mod h1:i1e4CEs42IrbidMUNTSNOKmeGPCOHVX9P3BvPxzyMtI=
 github.com/aler9/rtmp v0.0.0-20210403095203-3be4a5535927 h1:95mXJ5fUCYpBRdSOnLAQAdJHHKxxxJrVCiaqDi965YQ=
 github.com/aler9/rtmp v0.0.0-20210403095203-3be4a5535927/go.mod h1:vzuE21rowz+lT1NGsWbreIvYulgBpCGnQyeTyFblUHc=
 github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=

internal/core/rtmp_conn.go

@@ -331,8 +331,7 @@ func (c *rtmpConn) runRead(ctx context.Context) error {
 	var videoInitialPTS *time.Duration
 	videoFirstIDRFound := false
 	var videoFirstIDRPTS time.Duration
-	videoDTSExtractor := h264.NewDTSExtractor()
-	var videoSPS *h264.SPS
+	var videoDTSExtractor *h264.DTSExtractor
 
 	for {
 		item, ok := c.ringBuffer.Pull()
@@ -355,17 +354,28 @@ func (c *rtmpConn) runRead(ctx context.Context) error {
 			}
 			pts := data.h264PTS - *videoInitialPTS
 
+			idrPresent := h264.IDRPresent(data.h264NALUs)
+
 			// wait until we receive an IDR
 			if !videoFirstIDRFound {
-				if !h264.IDRPresent(data.h264NALUs) {
+				if !idrPresent {
 					continue
 				}
 
 				videoFirstIDRFound = true
 				videoFirstIDRPTS = pts
+				videoDTSExtractor = h264.NewDTSExtractor()
 			}
 
-			if h264.IDRPresent(data.h264NALUs) {
+			pts -= videoFirstIDRPTS
+
+			dts, err := videoDTSExtractor.Extract(data.h264NALUs, pts)
+			if err != nil {
+				return err
+			}
+
+			// insert a H264DecoderConfig before every IDR
+			if idrPresent {
 				sps := videoTrack.SPS()
 				pps := videoTrack.PPS()
 
@@ -389,13 +399,6 @@ func (c *rtmpConn) runRead(ctx context.Context) error {
 				if err != nil {
 					return err
 				}
-
-				var psps h264.SPS
-				err := psps.Unmarshal(sps)
-				if err != nil {
-					return err
-				}
-				videoSPS = &psps
 			}
 
 			avcc, err := h264.AVCCEncode(data.h264NALUs)
@@ -403,13 +406,6 @@ func (c *rtmpConn) runRead(ctx context.Context) error {
 				return err
 			}
 
-			pts -= videoFirstIDRPTS
-			dts, err := videoDTSExtractor.Extract(
-				data.h264NALUs, h264.IDRPresent(data.h264NALUs), pts, videoSPS)
-			if err != nil {
-				return err
-			}
-
 			c.conn.SetWriteDeadline(time.Now().Add(time.Duration(c.writeTimeout)))
 			err = c.conn.WritePacket(av.Packet{
 				Type:  av.H264,
@@ -568,7 +564,7 @@ func (c *rtmpConn) runPublish(ctx context.Context) error {
 					rres.stream.writeData(&data{
 						trackID:      videoTrackID,
 						rtp:          pkt,
-						ptsEqualsDTS: h264.IDRPresent(nalus),
+						ptsEqualsDTS: false,
 						h264NALUs:    nalus,
 						h264PTS:      pts,
 					})

internal/hls/muxer_test.go

@@ -13,8 +13,16 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+// baseline profile without POC
+var testSPS = []byte{
+	0x67, 0x42, 0xc0, 0x28, 0xd9, 0x00, 0x78, 0x02,
+	0x27, 0xe5, 0x84, 0x00, 0x00, 0x03, 0x00, 0x04,
+	0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, 0x60, 0xc9,
+	0x20,
+}
+
 func TestMuxerVideoAudio(t *testing.T) {
-	videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil)
+	videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil)
 	require.NoError(t, err)
 
 	audioTrack, err := gortsplib.NewTrackAAC(97, 2, 44100, 2, nil, 13, 3, 3)
@@ -33,7 +41,7 @@ func TestMuxerVideoAudio(t *testing.T) {
 
 	// group with IDR
 	err = m.WriteH264(2*time.Second, [][]byte{
-		{7}, // SPS
+		testSPS, // SPS
 		{8},     // PPS
 		{5},     // IDR
 	})
@@ -47,8 +55,7 @@ func TestMuxerVideoAudio(t *testing.T) {
 
 	// group without IDR
 	err = m.WriteH264(4*time.Second, [][]byte{
-		{6},
-		{7},
+		{1}, // non-IDR
 	})
 	require.NoError(t, err)
 
@@ -67,7 +74,7 @@ func TestMuxerVideoAudio(t *testing.T) {
 		"#EXT-X-VERSION:3\n"+
 		"#EXT-X-INDEPENDENT-SEGMENTS\n"+
 		"\n"+
-		"#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.010203,mp4a.40.2\"\n"+
+		"#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.42c028,mp4a.40.2\"\n"+
 		"stream.m3u8\n", string(byts))
 
 	byts, err = ioutil.ReadAll(m.File("stream.m3u8", "", "", "").Body)
@@ -126,8 +133,8 @@ func TestMuxerVideoAudio(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, &astits.Packet{
 		AdaptationField: &astits.PacketAdaptationField{
-			Length:                148,
-			StuffingLength:        141,
+			Length:                119,
+			StuffingLength:        112,
 			HasPCR:                true,
 			PCR:                   &astits.ClockReference{},
 			RandomAccessIndicator: true,
@@ -138,14 +145,16 @@ func TestMuxerVideoAudio(t *testing.T) {
 			PayloadUnitStartIndicator: true,
 			PID:                       256,
 		},
-		Payload: append([]byte{
-			0x00, 0x00, 0x01, 0xe0, 0x00, 0x00, 0x80, 0x80,
-			0x05, 0x21, 0x00, 0x03, 0x5f, 0x91,
-			0, 0, 0, 1, 9, 240, // AUD
-			0, 0, 0, 1, 7, // SPS
-			0, 0, 0, 1, 8, // PPS
-			0, 0, 0, 1, 5, // IDR
-		}, bytes.Repeat([]byte{0xff}, 0)...),
+		Payload: []byte{
+			0x00, 0x00, 0x01, 0xe0, 0x00, 0x00, 0x80, 0xc0,
+			0x0a, 0x31, 0x00, 0x05, 0x32, 0x81, 0x11, 0x00,
+			0x03, 0x19, 0x41, 0x00, 0x00, 0x00, 0x01, 0x09,
+			0xf0, 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0,
+			0x28, 0xd9, 0x00, 0x78, 0x02, 0x27, 0xe5, 0x84,
+			0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03,
+			0x00, 0xf0, 0x3c, 0x60, 0xc9, 0x20, 0x00, 0x00,
+			0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x01, 0x05,
+		},
 	}, pkt)
 
 	// PES (AAC)
@@ -163,18 +172,18 @@ func TestMuxerVideoAudio(t *testing.T) {
 			PayloadUnitStartIndicator: true,
 			PID:                       257,
 		},
-		Payload: append([]byte{
+		Payload: []byte{
 			0x00, 0x00, 0x01, 0xc0, 0x00, 0x1e, 0x80, 0x80,
-			0x05, 0x21, 0x00, 0x09, 0x1e, 0xb1, 0xff, 0xf1,
+			0x05, 0x21, 0x00, 0x09, 0xf1, 0xa1, 0xff, 0xf1,
 			0x50, 0x80, 0x01, 0x7f, 0xfc, 0x01, 0x02, 0x03,
 			0x04, 0xff, 0xf1, 0x50, 0x80, 0x01, 0x7f, 0xfc,
 			0x05, 0x06, 0x07, 0x08,
-		}, bytes.Repeat([]byte{0xff}, 0)...),
+		},
 	}, pkt)
 }
 
 func TestMuxerVideoOnly(t *testing.T) {
-	videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil)
+	videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil)
 	require.NoError(t, err)
 
 	m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil)
@@ -183,10 +192,9 @@ func TestMuxerVideoOnly(t *testing.T) {
 
 	// group with IDR
 	err = m.WriteH264(2*time.Second, [][]byte{
-		{5}, // IDR
-		{9}, // AUD
+		testSPS, // SPS
 		{8},     // PPS
-		{7}, // SPS
+		{5},     // IDR
 	})
 	require.NoError(t, err)
 
@@ -203,7 +211,7 @@ func TestMuxerVideoOnly(t *testing.T) {
 		"#EXT-X-VERSION:3\n"+
 		"#EXT-X-INDEPENDENT-SEGMENTS\n"+
 		"\n"+
-		"#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.010203\"\n"+
+		"#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.42c028\"\n"+
 		"stream.m3u8\n", string(byts))
 
 	byts, err = ioutil.ReadAll(m.File("stream.m3u8", "", "", "").Body)
@@ -346,7 +354,7 @@ func TestMuxerAudioOnly(t *testing.T) {
 }
 
 func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) {
-	videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil)
+	videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil)
 	require.NoError(t, err)
 
 	m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil)
@@ -354,10 +362,9 @@ func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) {
 
 	// group with IDR
 	err = m.WriteH264(2*time.Second, [][]byte{
-		{5}, // IDR
-		{9}, // AUD
+		testSPS, // SPS
 		{8},     // PPS
-		{7}, // SPS
+		{5},     // IDR
 	})
 	require.NoError(t, err)
 
@@ -368,7 +375,7 @@ func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) {
 }
 
 func TestMuxerMaxSegmentSize(t *testing.T) {
-	videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil)
+	videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil)
 	require.NoError(t, err)
 
 	m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 0, videoTrack, nil)
@@ -376,13 +383,14 @@ func TestMuxerMaxSegmentSize(t *testing.T) {
 	defer m.Close()
 
 	err = m.WriteH264(2*time.Second, [][]byte{
-		{5},
+		testSPS,
+		{5}, // IDR
 	})
 	require.EqualError(t, err, "reached maximum segment size")
 }
 
 func TestMuxerDoubleRead(t *testing.T) {
-	videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil)
+	videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil)
 	require.NoError(t, err)
 
 	m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil)
@@ -390,13 +398,14 @@ func TestMuxerDoubleRead(t *testing.T) {
 	defer m.Close()
 
 	err = m.WriteH264(0, [][]byte{
-		{5},
+		testSPS,
+		{5}, // IDR
 		{1},
 	})
 	require.NoError(t, err)
 
 	err = m.WriteH264(2*time.Second, [][]byte{
-		{5},
+		{5}, // IDR
 		{2},
 	})
 	require.NoError(t, err)

internal/hls/muxer_variant_fmp4_segmenter.go

@@ -54,11 +54,11 @@ type muxerVariantFMP4Segmenter struct {
 	onSegmentFinalized func(*muxerVariantFMP4Segment)
 	onPartFinalized    func(*muxerVariantFMP4Part)
 
-	currentSegment        *muxerVariantFMP4Segment
-	startPTS              time.Duration
-	videoSPS              []byte
-	videoSPSP             *h264.SPS
+	videoFirstIDRReceived bool
 	videoDTSExtractor     *h264.DTSExtractor
+	videoSPS              []byte
+	startPTS              time.Duration
+	currentSegment        *muxerVariantFMP4Segment
 	nextSegmentID         uint64
 	nextPartID            uint64
 	nextVideoSample       *fmp4VideoSample
@@ -88,7 +88,6 @@ func newMuxerVariantFMP4Segmenter(
 		audioTrack:         audioTrack,
 		onSegmentFinalized: onSegmentFinalized,
 		onPartFinalized:    onPartFinalized,
-		videoDTSExtractor:  h264.NewDTSExtractor(),
 		nextSegmentID:      uint64(segmentCount),
 		sampleDurations:    make(map[time.Duration]struct{}),
 	}
@@ -139,34 +138,24 @@ func (m *muxerVariantFMP4Segmenter) writeH264(pts time.Duration, nalus [][]byte)
 }
 
 func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) error {
-	// parse SPS
-	spsChanged := false
-	if sample.idrPresent {
-		videoSPS := m.videoTrack.SPS()
-		if m.videoSPS == nil || !bytes.Equal(m.videoSPS, videoSPS) {
-			spsChanged = true
-
-			var videoSPSP h264.SPS
-			err := videoSPSP.Unmarshal(videoSPS)
-			if err != nil {
-				return err
+	if !m.videoFirstIDRReceived {
+		// skip sample silently until we find one with an IDR
+		if !sample.idrPresent {
+			return nil
 		}
 
-			m.videoSPS = videoSPS
-			m.videoSPSP = &videoSPSP
-		}
+		m.videoFirstIDRReceived = true
+		m.videoDTSExtractor = h264.NewDTSExtractor()
+		m.videoSPS = append([]byte(nil), m.videoTrack.SPS()...)
 	}
 
 	// fill DTS
-	if m.videoSPSP != nil {
 	var err error
-		sample.dts, err = m.videoDTSExtractor.Extract(
-			sample.nalus, sample.idrPresent, sample.pts, m.videoSPSP)
+	sample.dts, err = m.videoDTSExtractor.Extract(sample.nalus, sample.pts)
 	if err != nil {
 		return err
 	}
 	sample.nalus = nil
-	}
 
 	sample.pts -= m.startPTS
 	sample.dts -= m.startPTS
@@ -183,11 +172,6 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) erro
 	now := time.Now()
 
 	if m.currentSegment == nil {
-		// skip groups silently until we find one with a IDR
-		if !sample.idrPresent {
-			return nil
-		}
-
 		// create first segment
 		m.currentSegment = newMuxerVariantFMP4Segment(
 			m.lowLatency,
@@ -210,13 +194,16 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) erro
 
 	m.adjustPartDuration(sample.duration())
 
-	err := m.currentSegment.writeH264(sample, m.adjustedPartDuration)
+	err = m.currentSegment.writeH264(sample, m.adjustedPartDuration)
 	if err != nil {
 		return err
 	}
 
 	// switch segment
 	if sample.next.idrPresent {
+		sps := m.videoTrack.SPS()
+		spsChanged := !bytes.Equal(m.videoSPS, sps)
+
 		if (sample.next.pts-m.currentSegment.startDTS) >= m.segmentDuration ||
 			spsChanged {
 			err := m.currentSegment.finalize(sample.next, nil)
@@ -241,6 +228,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) erro
 
 			// if SPS changed, reset adjusted part duration
 			if spsChanged {
+				m.videoSPS = append([]byte(nil), sps...)
 				m.firstSegmentFinalized = false
 				m.sampleDurations = make(map[time.Duration]struct{})
 			}

internal/hls/muxer_variant_mpegts_segment.go

@@ -110,15 +110,13 @@ func (t *muxerVariantMPEGTSSegment) writeH264(
 		MarkerBits: 2,
 	}
 
-	pts += mpegtsPTSDTSOffset
-
-	if dts == pts {
+	if dts == (pts + mpegtsPTSDTSOffset) {
 		oh.PTSDTSIndicator = astits.PTSDTSIndicatorOnlyPTS
-		oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPCROffset).Seconds() * 90000)}
+		oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPTSDTSOffset + mpegtsPCROffset).Seconds() * 90000)}
 	} else {
 		oh.PTSDTSIndicator = astits.PTSDTSIndicatorBothPresent
 		oh.DTS = &astits.ClockReference{Base: int64((dts + mpegtsPCROffset).Seconds() * 90000)}
-		oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPCROffset).Seconds() * 90000)}
+		oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPTSDTSOffset + mpegtsPCROffset).Seconds() * 90000)}
 	}
 
 	_, err = t.writeData(&astits.MuxerData{
@@ -182,8 +180,6 @@ func (t *muxerVariantMPEGTSSegment) writeAAC(
 		t.pcrSendCounter--
 	}
 
-	pts += mpegtsPTSDTSOffset
-
 	_, err = t.writeData(&astits.MuxerData{
 		PID:             257,
 		AdaptationField: af,
@@ -192,7 +188,7 @@ func (t *muxerVariantMPEGTSSegment) writeAAC(
 				OptionalHeader: &astits.PESOptionalHeader{
 					MarkerBits:      2,
 					PTSDTSIndicator: astits.PTSDTSIndicatorOnlyPTS,
-					PTS:             &astits.ClockReference{Base: int64((pts + mpegtsPCROffset).Seconds() * 90000)},
+					PTS:             &astits.ClockReference{Base: int64((pts + mpegtsPTSDTSOffset + mpegtsPCROffset).Seconds() * 90000)},
 				},
 				PacketLength: uint16(len(enc) + 8),
 				StreamID:     192, // audio

internal/hls/muxer_variant_mpegts_segmenter.go

@@ -28,7 +28,6 @@ type muxerVariantMPEGTSSegmenter struct {
 
 	writer            *astits.Muxer
 	currentSegment    *muxerVariantMPEGTSSegment
-	videoSPS          *h264.SPS
 	videoDTSExtractor *h264.DTSExtractor
 	startPCR          time.Time
 	startPTS          time.Duration
@@ -47,7 +46,6 @@ func newMuxerVariantMPEGTSSegmenter(
 		videoTrack:      videoTrack,
 		audioTrack:      audioTrack,
 		onSegmentReady:  onSegmentReady,
-		videoDTSExtractor: h264.NewDTSExtractor(),
 	}
 
 	m.writer = astits.NewMuxer(
@@ -94,6 +92,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(pts time.Duration, nalus [][]byt
 			m.videoTrack, m.audioTrack, m.writer.WriteData)
 		m.startPCR = now
 		m.startPTS = pts
+		m.videoDTSExtractor = h264.NewDTSExtractor()
 		pts = 0
 	} else {
 		pts -= m.startPTS
@@ -109,17 +108,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(pts time.Duration, nalus [][]byt
 		}
 	}
 
-	if idrPresent {
-		sps := m.videoTrack.SPS()
-		var psps h264.SPS
-		err := psps.Unmarshal(sps)
-		if err != nil {
-			return err
-		}
-		m.videoSPS = &psps
-	}
-
-	dts, err := m.videoDTSExtractor.Extract(nalus, idrPresent, pts, m.videoSPS)
+	dts, err := m.videoDTSExtractor.Extract(nalus, pts)
 	if err != nil {
 		return err
 	}