From 39da3003452e8d220a62a9663a239412d4f1bb79 Mon Sep 17 00:00:00 2001 From: aler9 <46489434+aler9@users.noreply.github.com> Date: Wed, 14 Dec 2022 15:41:59 +0100 Subject: [PATCH] update H265 track parameters when resolution, VPS, SPS or PPS change --- go.mod | 2 +- go.sum | 4 +- internal/core/formatprocessor_h264.go | 40 +++-- internal/core/formatprocessor_h265.go | 80 ++++++++- internal/core/rtsp_source_test.go | 241 +++++++++++++++++--------- 5 files changed, 262 insertions(+), 105 deletions(-) diff --git a/go.mod b/go.mod index ad938ceb..afd201a8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 github.com/abema/go-mp4 v0.8.0 - github.com/aler9/gortsplib/v2 v2.0.0-20221213201904-04d1de717768 + github.com/aler9/gortsplib/v2 v2.0.0-20221214135702-6141afcfc4c5 github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757 github.com/fsnotify/fsnotify v1.4.9 github.com/gin-gonic/gin v1.8.1 diff --git a/go.sum b/go.sum index a6ddc739..b2899cae 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aler9/gortsplib/v2 v2.0.0-20221213201904-04d1de717768 h1:Q7Qm3HjO8bYo0P2dPmKXYbfvpqhA9ZJuJ+TqN6UcdqI= -github.com/aler9/gortsplib/v2 v2.0.0-20221213201904-04d1de717768/go.mod h1:zJ+fWtakOMN6cKV169EMNVBLPTITArrJKu/fyM+dov8= +github.com/aler9/gortsplib/v2 v2.0.0-20221214135702-6141afcfc4c5 h1:QHF5GXc8Q23sCYYAfD24mwOJt1CX9by3KY5gYolYEaw= +github.com/aler9/gortsplib/v2 v2.0.0-20221214135702-6141afcfc4c5/go.mod h1:zJ+fWtakOMN6cKV169EMNVBLPTITArrJKu/fyM+dov8= github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4= github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U= github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8= diff --git a/internal/core/formatprocessor_h264.go b/internal/core/formatprocessor_h264.go index 00cf0d07..1658f000 100644 --- a/internal/core/formatprocessor_h264.go +++ b/internal/core/formatprocessor_h264.go @@ -10,24 +10,9 @@ import ( "github.com/pion/rtp" ) -type dataH264 struct { - rtpPackets []*rtp.Packet - ntp time.Time - pts time.Duration - nalus [][]byte -} - -func (d *dataH264) getRTPPackets() []*rtp.Packet { - return d.rtpPackets -} - -func (d *dataH264) getNTP() time.Time { - return d.ntp -} - // extract SPS and PPS without decoding RTP packets func rtpH264ExtractSPSPPS(pkt *rtp.Packet) ([]byte, []byte) { - if len(pkt.Payload) == 0 { + if len(pkt.Payload) < 1 { return nil, nil } @@ -40,7 +25,7 @@ func rtpH264ExtractSPSPPS(pkt *rtp.Packet) ([]byte, []byte) { case h264.NALUTypePPS: return nil, pkt.Payload - case 24: // STAP-A + case h264.NALUTypeSTAPA: payload := pkt.Payload[1:] var sps []byte var pps []byte @@ -53,10 +38,14 @@ func rtpH264ExtractSPSPPS(pkt *rtp.Packet) ([]byte, []byte) { size := uint16(payload[0])<<8 | uint16(payload[1]) payload = payload[2:] - if size == 0 || int(size) > len(payload) { + if size == 0 { break } + if int(size) > len(payload) { + return nil, nil + } + nalu := payload[:size] payload = payload[size:] @@ -78,6 +67,21 @@ func rtpH264ExtractSPSPPS(pkt *rtp.Packet) ([]byte, []byte) { } } +type dataH264 struct { + rtpPackets []*rtp.Packet + ntp time.Time + pts time.Duration + nalus [][]byte +} + +func (d *dataH264) getRTPPackets() []*rtp.Packet { + return d.rtpPackets +} + +func (d *dataH264) getNTP() time.Time { + return d.ntp +} + type formatProcessorH264 struct { format *format.H264 diff --git a/internal/core/formatprocessor_h265.go b/internal/core/formatprocessor_h265.go index e16dfe12..cb30cd4a 100644 --- a/internal/core/formatprocessor_h265.go +++ b/internal/core/formatprocessor_h265.go @@ -1,14 +1,80 @@ package core import ( + "bytes" "fmt" "time" "github.com/aler9/gortsplib/v2/pkg/format" "github.com/aler9/gortsplib/v2/pkg/formatdecenc/rtph265" + "github.com/aler9/gortsplib/v2/pkg/h265" "github.com/pion/rtp" ) +// extract VPS, SPS and PPS without decoding RTP packets +func rtpH265ExtractVPSSPSPPS(pkt *rtp.Packet) ([]byte, []byte, []byte) { + if len(pkt.Payload) < 2 { + return nil, nil, nil + } + + typ := h265.NALUType((pkt.Payload[0] >> 1) & 0b111111) + + switch typ { + case h265.NALUTypeVPS: + return pkt.Payload, nil, nil + + case h265.NALUTypeSPS: + return nil, pkt.Payload, nil + + case h265.NALUTypePPS: + return nil, nil, pkt.Payload + + case h265.NALUTypeAggregationUnit: + payload := pkt.Payload[2:] + var vps []byte + var sps []byte + var pps []byte + + for len(payload) > 0 { + if len(payload) < 2 { + break + } + + size := uint16(payload[0])<<8 | uint16(payload[1]) + payload = payload[2:] + + if size == 0 { + break + } + + if int(size) > len(payload) { + return nil, nil, nil + } + + nalu := payload[:size] + payload = payload[size:] + + typ = h265.NALUType((pkt.Payload[0] >> 1) & 0b111111) + + switch typ { + case h265.NALUTypeVPS: + vps = nalu + + case h265.NALUTypeSPS: + sps = nalu + + case h265.NALUTypePPS: + pps = nalu + } + } + + return vps, sps, pps + + default: + return nil, nil, nil + } +} + type dataH265 struct { rtpPackets []*rtp.Packet ntp time.Time @@ -47,7 +113,19 @@ func newFormatProcessorH265( } func (t *formatProcessorH265) updateTrackParametersFromRTPPacket(pkt *rtp.Packet) { - // TODO: extract VPS, SPS, PPS and set them into the track + vps, sps, pps := rtpH265ExtractVPSSPSPPS(pkt) + + if vps != nil && !bytes.Equal(vps, t.format.SafeVPS()) { + t.format.SafeSetVPS(vps) + } + + if sps != nil && !bytes.Equal(sps, t.format.SafeSPS()) { + t.format.SafeSetSPS(sps) + } + + if pps != nil && !bytes.Equal(pps, t.format.SafePPS()) { + t.format.SafeSetPPS(pps) + } } func (t *formatProcessorH265) updateTrackParametersFromNALUs(nalus [][]byte) { diff --git a/internal/core/rtsp_source_test.go b/internal/core/rtsp_source_test.go index ec935f4b..9bc697f8 100644 --- a/internal/core/rtsp_source_test.go +++ b/internal/core/rtsp_source_test.go @@ -14,6 +14,7 @@ import ( "github.com/aler9/gortsplib/v2/pkg/base" "github.com/aler9/gortsplib/v2/pkg/conn" "github.com/aler9/gortsplib/v2/pkg/format" + "github.com/aler9/gortsplib/v2/pkg/h265" "github.com/aler9/gortsplib/v2/pkg/headers" "github.com/aler9/gortsplib/v2/pkg/media" "github.com/aler9/gortsplib/v2/pkg/url" @@ -236,119 +237,193 @@ func TestRTSPSourceNoPassword(t *testing.T) { } func TestRTSPSourceDynamicH264Params(t *testing.T) { - forma := &format.H264{ - PayloadTyp: 96, - PacketizationMode: 1, - } - medi := &media.Media{ - Type: media.TypeVideo, - Formats: []format.Format{forma}, - } - stream := gortsplib.NewServerStream(media.Medias{medi}) - defer stream.Close() + checkTrack := func(t *testing.T, forma format.Format) { + c := gortsplib.Client{} - s := gortsplib.Server{ - Handler: &testServer{ - onDescribe: func(ctx *gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) { - return &base.Response{ - StatusCode: base.StatusOK, - }, stream, nil - }, - onSetup: func(ctx *gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) { - return &base.Response{ - StatusCode: base.StatusOK, - }, stream, nil - }, - onPlay: func(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) { - return &base.Response{ - StatusCode: base.StatusOK, - }, nil - }, - }, - RTSPAddress: "127.0.0.1:8555", + u, err := url.Parse("rtsp://127.0.0.1:8554/proxied") + require.NoError(t, err) + + err = c.Start(u.Scheme, u.Host) + require.NoError(t, err) + defer c.Close() + + medias, _, _, err := c.Describe(u) + require.NoError(t, err) + + forma1 := medias[0].Formats[0] + require.Equal(t, forma, forma1) } - err := s.Start() - require.NoError(t, err) - defer s.Wait() - defer s.Close() - p, ok := newInstance("rtmpDisable: yes\n" + - "hlsDisable: yes\n" + - "paths:\n" + - " proxied:\n" + - " source: rtsp://127.0.0.1:8555/teststream\n") - require.Equal(t, true, ok) - defer p.Close() + t.Run("h264", func(t *testing.T) { + forma := &format.H264{ + PayloadTyp: 96, + PacketizationMode: 1, + } + medi := &media.Media{ + Type: media.TypeVideo, + Formats: []format.Format{forma}, + } + stream := gortsplib.NewServerStream(media.Medias{medi}) + defer stream.Close() - time.Sleep(1 * time.Second) + s := gortsplib.Server{ + Handler: &testServer{ + onDescribe: func(ctx *gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{ + StatusCode: base.StatusOK, + }, stream, nil + }, + onSetup: func(ctx *gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{ + StatusCode: base.StatusOK, + }, stream, nil + }, + onPlay: func(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) { + return &base.Response{ + StatusCode: base.StatusOK, + }, nil + }, + }, + RTSPAddress: "127.0.0.1:8555", + } + err := s.Start() + require.NoError(t, err) + defer s.Wait() + defer s.Close() - enc := forma.CreateEncoder() + p, ok := newInstance("rtmpDisable: yes\n" + + "hlsDisable: yes\n" + + "paths:\n" + + " proxied:\n" + + " source: rtsp://127.0.0.1:8555/teststream\n") + require.Equal(t, true, ok) + defer p.Close() - pkts, err := enc.Encode([][]byte{{7, 1, 2, 3}}, 0) // SPS - require.NoError(t, err) - stream.WritePacketRTP(medi, pkts[0]) + time.Sleep(1 * time.Second) - pkts, err = enc.Encode([][]byte{{8}}, 0) // PPS - require.NoError(t, err) - stream.WritePacketRTP(medi, pkts[0]) + enc := forma.CreateEncoder() + + pkts, err := enc.Encode([][]byte{{7, 1, 2, 3}}, 0) // SPS + require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) - time.Sleep(500 * time.Millisecond) + pkts, err = enc.Encode([][]byte{{8}}, 0) // PPS + require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) - func() { - c := gortsplib.Client{} + checkTrack(t, &format.H264{ + PayloadTyp: 96, + PacketizationMode: 1, + SPS: []byte{7, 1, 2, 3}, + PPS: []byte{8}, + }) - u, err := url.Parse("rtsp://127.0.0.1:8554/proxied") + pkts, err = enc.Encode([][]byte{{7, 4, 5, 6}}, 0) // SPS require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) - err = c.Start(u.Scheme, u.Host) + pkts, err = enc.Encode([][]byte{{8, 1}}, 0) // PPS require.NoError(t, err) - defer c.Close() + stream.WritePacketRTP(medi, pkts[0]) - medias, _, _, err := c.Describe(u) + checkTrack(t, &format.H264{ + PayloadTyp: 96, + PacketizationMode: 1, + SPS: []byte{7, 4, 5, 6}, + PPS: []byte{8, 1}, + }) + }) + + t.Run("h265", func(t *testing.T) { + forma := &format.H265{ + PayloadTyp: 96, + } + medi := &media.Media{ + Type: media.TypeVideo, + Formats: []format.Format{forma}, + } + stream := gortsplib.NewServerStream(media.Medias{medi}) + defer stream.Close() + + s := gortsplib.Server{ + Handler: &testServer{ + onDescribe: func(ctx *gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{ + StatusCode: base.StatusOK, + }, stream, nil + }, + onSetup: func(ctx *gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) { + return &base.Response{ + StatusCode: base.StatusOK, + }, stream, nil + }, + onPlay: func(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) { + return &base.Response{ + StatusCode: base.StatusOK, + }, nil + }, + }, + RTSPAddress: "127.0.0.1:8555", + } + err := s.Start() require.NoError(t, err) + defer s.Wait() + defer s.Close() - h264Track := medias[0].Formats[0].(*format.H264) - require.Equal(t, []byte{7, 1, 2, 3}, h264Track.SafeSPS()) - require.Equal(t, []byte{8}, h264Track.SafePPS()) - }() + p, ok := newInstance("rtmpDisable: yes\n" + + "hlsDisable: yes\n" + + "paths:\n" + + " proxied:\n" + + " source: rtsp://127.0.0.1:8555/teststream\n") + require.Equal(t, true, ok) + defer p.Close() - pkts, err = enc.Encode([][]byte{{7, 4, 5, 6}}, 0) // SPS - require.NoError(t, err) - stream.WritePacketRTP(medi, pkts[0]) + time.Sleep(1 * time.Second) - pkts, err = enc.Encode([][]byte{{8, 1}}, 0) // PPS - require.NoError(t, err) - stream.WritePacketRTP(medi, pkts[0]) + enc := forma.CreateEncoder() - time.Sleep(500 * time.Millisecond) + pkts, err := enc.Encode([][]byte{{byte(h265.NALUTypeVPS) << 1, 1, 2, 3}}, 0) + require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) - func() { - c := gortsplib.Client{} + pkts, err = enc.Encode([][]byte{{byte(h265.NALUTypeSPS) << 1, 4, 5, 6}}, 0) + require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) - u, err := url.Parse("rtsp://127.0.0.1:8554/proxied") + pkts, err = enc.Encode([][]byte{{byte(h265.NALUTypePPS) << 1, 7, 8, 9}}, 0) require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) - err = c.Start(u.Scheme, u.Host) + checkTrack(t, &format.H265{ + PayloadTyp: 96, + VPS: []byte{byte(h265.NALUTypeVPS) << 1, 1, 2, 3}, + SPS: []byte{byte(h265.NALUTypeSPS) << 1, 4, 5, 6}, + PPS: []byte{byte(h265.NALUTypePPS) << 1, 7, 8, 9}, + }) + + pkts, err = enc.Encode([][]byte{{byte(h265.NALUTypeVPS) << 1, 10, 11, 12}}, 0) require.NoError(t, err) - defer c.Close() + stream.WritePacketRTP(medi, pkts[0]) - medias, _, _, err := c.Describe(u) + pkts, err = enc.Encode([][]byte{{byte(h265.NALUTypeSPS) << 1, 13, 14, 15}}, 0) require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) - h264Track := medias[0].Formats[0].(*format.H264) - require.Equal(t, []byte{7, 4, 5, 6}, h264Track.SafeSPS()) - require.Equal(t, []byte{8, 1}, h264Track.SafePPS()) - }() + pkts, err = enc.Encode([][]byte{{byte(h265.NALUTypePPS) << 1, 16, 17, 18}}, 0) + require.NoError(t, err) + stream.WritePacketRTP(medi, pkts[0]) + + checkTrack(t, &format.H265{ + PayloadTyp: 96, + VPS: []byte{byte(h265.NALUTypeVPS) << 1, 10, 11, 12}, + SPS: []byte{byte(h265.NALUTypeSPS) << 1, 13, 14, 15}, + PPS: []byte{byte(h265.NALUTypePPS) << 1, 16, 17, 18}, + }) + }) } func TestRTSPSourceRemovePadding(t *testing.T) { - medi := &media.Media{ - Type: media.TypeVideo, - Formats: []format.Format{&format.H264{ - PayloadTyp: 96, - PacketizationMode: 1, - }}, - } + medi := testMediaH264 stream := gortsplib.NewServerStream(media.Medias{medi}) defer stream.Close()