add SPS and PTS before IDRs of all incoming H264 streams; stop filtering H264 inside single protocols

internal/core/rtmp_conn.go

@@ -342,36 +342,11 @@ func (c *rtmpConn) runRead(ctx context.Context) error {
 				continue
 			}
 
-			var filteredNALUs [][]byte
-
-			for _, nalu := range pair.data.h264NALUs {
-				typ := h264.NALUType(nalu[0] & 0x1F)
-
-				switch typ {
-				case h264.NALUTypeSPS, h264.NALUTypePPS:
-					// added automatically before every IDR
-					continue
-
-				case h264.NALUTypeAccessUnitDelimiter:
-					// not needed
-					continue
-
-				case h264.NALUTypeIDR:
-					// add SPS and PPS before every IDR
-					// TODO: send H264DecoderConfig instead of NALUs?
-					filteredNALUs = append(filteredNALUs, videoTrack.SPS(), videoTrack.PPS())
-				}
-
-				filteredNALUs = append(filteredNALUs, nalu)
-			}
-
-			if filteredNALUs == nil {
-				continue
-			}
+			// TODO: send H264DecoderConfig instead of NALUs?
 
 			// wait until we receive an IDR
 			if !videoFirstIDRFound {
-				if !h264.IDRPresent(filteredNALUs) {
+				if !h264.IDRPresent(pair.data.h264NALUs) {
 					continue
 				}
 
@@ -380,7 +355,7 @@ func (c *rtmpConn) runRead(ctx context.Context) error {
 				videoDTSEst = h264.NewDTSEstimator()
 			}
 
-			data, err := h264.EncodeAVCC(filteredNALUs)
+			data, err := h264.EncodeAVCC(pair.data.h264NALUs)
 			if err != nil {
 				return err
 			}
@@ -559,25 +534,9 @@ func (c *rtmpConn) runPublish(ctx context.Context) error {
 				return err
 			}
 
-			var outNALUs [][]byte
-
-			for _, nalu := range nalus {
-				typ := h264.NALUType(nalu[0] & 0x1F)
-				if typ == h264.NALUTypeAccessUnitDelimiter {
-					// not needed
-					continue
-				}
-
-				outNALUs = append(outNALUs, nalu)
-			}
-
-			if len(outNALUs) == 0 {
-				continue
-			}
-
 			pts := pkt.Time + pkt.CTime
 
-			pkts, err := h264Encoder.Encode(outNALUs, pts)
+			pkts, err := h264Encoder.Encode(nalus, pts)
 			if err != nil {
 				return fmt.Errorf("error while encoding H264: %v", err)
 			}
@@ -592,8 +551,8 @@ func (c *rtmpConn) runPublish(ctx context.Context) error {
 				} else {
 					rres.stream.writeData(videoTrackID, &data{
 						rtp:          pkt,
-						ptsEqualsDTS: h264.IDRPresent(outNALUs),
-						h264NALUs:    outNALUs,
+						ptsEqualsDTS: h264.IDRPresent(nalus),
+						h264NALUs:    nalus,
 						h264PTS:      pts,
 					})
 				}

internal/core/rtmp_source.go

@@ -184,21 +184,9 @@ func (s *rtmpSource) runInner() bool {
 								return err
 							}
 
-							var outNALUs [][]byte
-							for _, nalu := range nalus {
-								// remove SPS, PPS and AUD, not needed by RTSP / RTMP
-								typ := h264.NALUType(nalu[0] & 0x1F)
-								switch typ {
-								case h264.NALUTypeSPS, h264.NALUTypePPS, h264.NALUTypeAccessUnitDelimiter:
-									continue
-								}
-
-								outNALUs = append(outNALUs, nalu)
-							}
-
 							pts := pkt.Time + pkt.CTime
 
-							pkts, err := h264Encoder.Encode(outNALUs, pts)
+							pkts, err := h264Encoder.Encode(nalus, pts)
 							if err != nil {
 								return fmt.Errorf("error while encoding H264: %v", err)
 							}
@@ -213,8 +201,8 @@ func (s *rtmpSource) runInner() bool {
 								} else {
 									res.stream.writeData(videoTrackID, &data{
 										rtp:          pkt,
-										ptsEqualsDTS: h264.IDRPresent(outNALUs),
-										h264NALUs:    outNALUs,
+										ptsEqualsDTS: h264.IDRPresent(nalus),
+										h264NALUs:    nalus,
 										h264PTS:      pts,
 									})
 								}

internal/core/stream.go

@@ -98,10 +98,39 @@ func (s *stream) updateH264TrackParameters(h264track *gortsplib.TrackH264, nalus
 	}
 }
 
+// remux is needed to
+// - fix corrupted streams
+// - make streams compatible with all protocols
+func (s *stream) remuxH264NALUs(h264track *gortsplib.TrackH264, data *data) {
+	var filteredNALUs [][]byte //nolint:prealloc
+
+	for _, nalu := range data.h264NALUs {
+		typ := h264.NALUType(nalu[0] & 0x1F)
+		switch typ {
+		case h264.NALUTypeSPS, h264.NALUTypePPS:
+			// remove since they're automatically added before every IDR
+			continue
+
+		case h264.NALUTypeAccessUnitDelimiter:
+			// remove since it is not needed
+			continue
+
+		case h264.NALUTypeIDR:
+			// add SPS and PPS before every IDR
+			filteredNALUs = append(filteredNALUs, h264track.SPS(), h264track.PPS())
+		}
+
+		filteredNALUs = append(filteredNALUs, nalu)
+	}
+
+	data.h264NALUs = filteredNALUs
+}
+
 func (s *stream) writeData(trackID int, data *data) {
 	track := s.rtspStream.Tracks()[trackID]
 	if h264track, ok := track.(*gortsplib.TrackH264); ok {
 		s.updateH264TrackParameters(h264track, data.h264NALUs)
+		s.remuxH264NALUs(h264track, data)
 	}
 
 	// forward to RTSP readers

internal/hls/client_video_processor.go

@@ -89,24 +89,7 @@ func (p *clientVideoProcessor) doProcess(
 		return nil
 	}
 
-	outNALUs := make([][]byte, 0, len(nalus))
-
-	for _, nalu := range nalus {
-		typ := h264.NALUType(nalu[0] & 0x1F)
-
-		if typ == h264.NALUTypeAccessUnitDelimiter {
-			// remove since it's not needed
-			continue
-		}
-
-		outNALUs = append(outNALUs, nalu)
-	}
-
-	if len(outNALUs) == 0 {
-		return nil
-	}
-
-	p.onData(pts, outNALUs)
+	p.onData(pts, nalus)
 	return nil
 }
 

internal/hls/muxer_ts_generator.go

@@ -115,11 +115,9 @@ func (m *muxerTSGenerator) writeH264(pts time.Duration, nalus [][]byte) error {
 	dts := m.videoDTSEst.Feed(pts)
 
 	// prepend an AUD. This is required by video.js and iOS
-	filteredNALUs := [][]byte{
-		{byte(h264.NALUTypeAccessUnitDelimiter), 240},
-	}
+	nalus = append([][]byte{{byte(h264.NALUTypeAccessUnitDelimiter), 240}}, nalus...)
 
-	enc, err := h264.EncodeAnnexB(filteredNALUs)
+	enc, err := h264.EncodeAnnexB(nalus)
 	if err != nil {
 		if m.currentSegment.buf.Len() > 0 {
 			m.streamPlaylist.pushSegment(m.currentSegment)