From bf691d168015702f32803cf93a8ced0140e9aedb Mon Sep 17 00:00:00 2001
From: Alessandro Ros <aler9.dev@gmail.com>
Date: Thu, 23 Feb 2023 14:46:26 +0100
Subject: [PATCH] formatprocessor: prevent generating empty H264/H265 RTP
 packets (#1505)

---
 internal/formatprocessor/h264.go      | 12 ++++++++----
 internal/formatprocessor/h264_test.go | 22 ++++++++++++++++++++++
 internal/formatprocessor/h265.go      | 12 ++++++++----
 internal/formatprocessor/h265_test.go | 22 ++++++++++++++++++++++
 internal/formatprocessor/processor.go |  2 +-
 5 files changed, 61 insertions(+), 9 deletions(-)

diff --git a/internal/formatprocessor/h264.go b/internal/formatprocessor/h264.go
index 3c628836..240f495d 100644
--- a/internal/formatprocessor/h264.go
+++ b/internal/formatprocessor/h264.go
@@ -259,11 +259,15 @@ func (t *formatProcessorH264) Process(dat Data, hasNonRTSPReaders bool) error {
 		tdata.AU = t.remuxAccessUnit(tdata.AU)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
-	if err != nil {
-		return err
+	if len(tdata.AU) != 0 {
+		pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
+		if err != nil {
+			return err
+		}
+		tdata.RTPPackets = pkts
+	} else {
+		tdata.RTPPackets = nil
 	}
 
-	tdata.RTPPackets = pkts
 	return nil
 }
diff --git a/internal/formatprocessor/h264_test.go b/internal/formatprocessor/h264_test.go
index fa1f3b61..f796072a 100644
--- a/internal/formatprocessor/h264_test.go
+++ b/internal/formatprocessor/h264_test.go
@@ -151,3 +151,25 @@ func TestH264OversizedPackets(t *testing.T) {
 		},
 	}, out)
 }
+
+func TestH264EmptyPacket(t *testing.T) {
+	forma := &format.H264{
+		PayloadTyp:        96,
+		PacketizationMode: 1,
+	}
+
+	p, err := New(forma, true)
+	require.NoError(t, err)
+
+	unit := &DataH264{
+		AU: [][]byte{
+			{0x07, 0x01, 0x02, 0x03}, // SPS
+			{0x08, 0x01, 0x02},       // PPS
+		},
+	}
+
+	p.Process(unit, false)
+
+	// if all NALUs have been removed, no RTP packets must be generated.
+	require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets)
+}
diff --git a/internal/formatprocessor/h265.go b/internal/formatprocessor/h265.go
index e634f68f..d420820c 100644
--- a/internal/formatprocessor/h265.go
+++ b/internal/formatprocessor/h265.go
@@ -280,11 +280,15 @@ func (t *formatProcessorH265) Process(dat Data, hasNonRTSPReaders bool) error {
 		tdata.AU = t.remuxAccessUnit(tdata.AU)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
-	if err != nil {
-		return err
+	if len(tdata.AU) != 0 {
+		pkts, err := t.encoder.Encode(tdata.AU, tdata.PTS)
+		if err != nil {
+			return err
+		}
+		tdata.RTPPackets = pkts
+	} else {
+		tdata.RTPPackets = nil
 	}
 
-	tdata.RTPPackets = pkts
 	return nil
 }
diff --git a/internal/formatprocessor/h265_test.go b/internal/formatprocessor/h265_test.go
index e3c70c08..18832e8c 100644
--- a/internal/formatprocessor/h265_test.go
+++ b/internal/formatprocessor/h265_test.go
@@ -144,3 +144,25 @@ func TestH265OversizedPackets(t *testing.T) {
 		},
 	}, out)
 }
+
+func TestH265EmptyPacket(t *testing.T) {
+	forma := &format.H265{
+		PayloadTyp: 96,
+	}
+
+	p, err := New(forma, true)
+	require.NoError(t, err)
+
+	unit := &DataH265{
+		AU: [][]byte{
+			{byte(h265.NALUType_VPS_NUT) << 1, 10, 11, 12}, // VPS
+			{byte(h265.NALUType_SPS_NUT) << 1, 13, 14, 15}, // SPS
+			{byte(h265.NALUType_PPS_NUT) << 1, 16, 17, 18}, // PPS
+		},
+	}
+
+	p.Process(unit, false)
+
+	// if all NALUs have been removed, no RTP packets must be generated.
+	require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets)
+}
diff --git a/internal/formatprocessor/processor.go b/internal/formatprocessor/processor.go
index 06df6890..07ed56a3 100644
--- a/internal/formatprocessor/processor.go
+++ b/internal/formatprocessor/processor.go
@@ -17,7 +17,7 @@ type Data interface {
 
 // Processor allows to cleanup and normalize streams.
 type Processor interface {
-	// cleanups and normalizes a data unit.
+	// clears and normalizes a data unit.
 	Process(Data, bool) error
 }