Browse Source

Decode streams once and only when needed (#1218)

* split data into specialized structs

* move MPEG4-audio decoding into streamTrack

* restore video/audio synchronization in HLS muxer and RTMP server

* log decode errors

* move H264 decoding and re-encoding here from gortsplib

* add tests

* update gortsplib
pull/1212/head
Alessandro Ros 3 years ago committed by GitHub
parent
commit
0943b269ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      go.mod
  2. 4
      go.sum
  3. 65
      internal/core/data.go
  4. 126
      internal/core/hls_muxer.go
  5. 19
      internal/core/hls_source.go
  6. 26
      internal/core/hls_source_test.go
  7. 2
      internal/core/reader.go
  8. 7
      internal/core/rpicamera_source.go
  9. 101
      internal/core/rtmp_conn.go
  10. 19
      internal/core/rtmp_source.go
  11. 31
      internal/core/rtsp_session.go
  12. 29
      internal/core/rtsp_source.go
  13. 353
      internal/core/rtsp_source_test.go
  14. 29
      internal/core/stream.go
  15. 10
      internal/core/streamtrack.go
  16. 32
      internal/core/streamtrack_generic.go
  17. 168
      internal/core/streamtrack_h264.go
  18. 75
      internal/core/streamtrack_mpeg4audio.go
  19. 8
      internal/hls/muxer_variant_fmp4_segmenter.go

2
go.mod

@ -5,7 +5,7 @@ go 1.18
require ( require (
code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
github.com/abema/go-mp4 v0.8.0 github.com/abema/go-mp4 v0.8.0
github.com/aler9/gortsplib v0.0.0-20221101102023-dbb6934a3c3e github.com/aler9/gortsplib v0.0.0-20221102164639-d3c23a849c83
github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757 github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757
github.com/fsnotify/fsnotify v1.4.9 github.com/fsnotify/fsnotify v1.4.9
github.com/gin-gonic/gin v1.8.1 github.com/gin-gonic/gin v1.8.1

4
go.sum

@ -6,8 +6,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/aler9/gortsplib v0.0.0-20221101102023-dbb6934a3c3e h1:x+EHN8/YHjG6NQM59WG+fdPmozyIarDZgJZymNbDmFE= github.com/aler9/gortsplib v0.0.0-20221102164639-d3c23a849c83 h1:Qn/TL5+Nm4g+IgQ1DODtu6oCve0plBiJsprbnLG3yfQ=
github.com/aler9/gortsplib v0.0.0-20221101102023-dbb6934a3c3e/go.mod h1:BOWNZ/QBkY/eVcRqUzJbPFEsRJshwxaxBT01K260Jeo= github.com/aler9/gortsplib v0.0.0-20221102164639-d3c23a849c83/go.mod h1:BOWNZ/QBkY/eVcRqUzJbPFEsRJshwxaxBT01K260Jeo=
github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4= github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=
github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U= github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U=
github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8= github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=

65
internal/core/data.go

@ -7,20 +7,65 @@ import (
) )
// data is the data unit routed across the server. // data is the data unit routed across the server.
// it must contain one or more of the following: type data interface {
// - a single RTP packet getTrackID() int
// - a group of H264 NALUs (grouped by timestamp) getRTPPackets() []*rtp.Packet
// - a single AAC AU getPTSEqualsDTS() bool
type data struct { }
trackID int
type dataGeneric struct {
trackID int
rtpPackets []*rtp.Packet
ptsEqualsDTS bool
}
rtpPacket *rtp.Packet func (d *dataGeneric) getTrackID() int {
return d.trackID
}
// timing func (d *dataGeneric) getRTPPackets() []*rtp.Packet {
return d.rtpPackets
}
func (d *dataGeneric) getPTSEqualsDTS() bool {
return d.ptsEqualsDTS
}
type dataH264 struct {
trackID int
rtpPackets []*rtp.Packet
ptsEqualsDTS bool ptsEqualsDTS bool
pts time.Duration pts time.Duration
nalus [][]byte
}
h264NALUs [][]byte func (d *dataH264) getTrackID() int {
return d.trackID
}
func (d *dataH264) getRTPPackets() []*rtp.Packet {
return d.rtpPackets
}
func (d *dataH264) getPTSEqualsDTS() bool {
return d.ptsEqualsDTS
}
type dataMPEG4Audio struct {
trackID int
rtpPackets []*rtp.Packet
pts time.Duration
aus [][]byte
}
func (d *dataMPEG4Audio) getTrackID() int {
return d.trackID
}
func (d *dataMPEG4Audio) getRTPPackets() []*rtp.Packet {
return d.rtpPackets
}
mpeg4AudioAU []byte func (d *dataMPEG4Audio) getPTSEqualsDTS() bool {
return true
} }

126
internal/core/hls_muxer.go

@ -14,7 +14,6 @@ import (
"github.com/aler9/gortsplib" "github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/mpeg4audio" "github.com/aler9/gortsplib/pkg/mpeg4audio"
"github.com/aler9/gortsplib/pkg/ringbuffer" "github.com/aler9/gortsplib/pkg/ringbuffer"
"github.com/aler9/gortsplib/pkg/rtpmpeg4audio"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/aler9/rtsp-simple-server/internal/conf" "github.com/aler9/rtsp-simple-server/internal/conf"
@ -295,7 +294,6 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
videoTrackID := -1 videoTrackID := -1
var audioTrack *gortsplib.TrackMPEG4Audio var audioTrack *gortsplib.TrackMPEG4Audio
audioTrackID := -1 audioTrackID := -1
var aacDecoder *rtpmpeg4audio.Decoder
for i, track := range res.stream.tracks() { for i, track := range res.stream.tracks() {
switch tt := track.(type) { switch tt := track.(type) {
@ -314,13 +312,6 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
audioTrack = tt audioTrack = tt
audioTrackID = i audioTrackID = i
aacDecoder = &rtpmpeg4audio.Decoder{
SampleRate: tt.Config.SampleRate,
SizeLength: tt.SizeLength,
IndexLength: tt.IndexLength,
IndexDeltaLength: tt.IndexDeltaLength,
}
aacDecoder.Init()
} }
} }
@ -362,53 +353,12 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
writerDone := make(chan error) writerDone := make(chan error)
go func() { go func() {
writerDone <- func() error { writerDone <- m.runWriter(
var videoInitialPTS *time.Duration videoTrack,
videoTrackID,
for { audioTrack,
item, ok := m.ringBuffer.Pull() audioTrackID,
if !ok { )
return fmt.Errorf("terminated")
}
data := item.(*data)
if videoTrack != nil && data.trackID == videoTrackID {
if data.h264NALUs == nil {
continue
}
if videoInitialPTS == nil {
v := data.pts
videoInitialPTS = &v
}
pts := data.pts - *videoInitialPTS
err = m.muxer.WriteH264(time.Now(), pts, data.h264NALUs)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
} else if audioTrack != nil && data.trackID == audioTrackID {
aus, pts, err := aacDecoder.Decode(data.rtpPacket)
if err != nil {
if err != rtpmpeg4audio.ErrMorePacketsNeeded {
m.log(logger.Warn, "unable to decode audio track: %v", err)
}
continue
}
for i, au := range aus {
err = m.muxer.WriteAAC(
time.Now(),
pts+time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/time.Duration(audioTrack.ClockRate()),
au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
}
}
}
}()
}() }()
closeCheckTicker := time.NewTicker(closeCheckPeriod) closeCheckTicker := time.NewTicker(closeCheckPeriod)
@ -435,6 +385,68 @@ func (m *hlsMuxer) runInner(innerCtx context.Context, innerReady chan struct{})
} }
} }
func (m *hlsMuxer) runWriter(
videoTrack *gortsplib.TrackH264,
videoTrackID int,
audioTrack *gortsplib.TrackMPEG4Audio,
audioTrackID int,
) error {
videoStartPTSFilled := false
var videoStartPTS time.Duration
audioStartPTSFilled := false
var audioStartPTS time.Duration
for {
item, ok := m.ringBuffer.Pull()
if !ok {
return fmt.Errorf("terminated")
}
data := item.(data)
if videoTrack != nil && data.getTrackID() == videoTrackID {
tdata := data.(*dataH264)
if tdata.nalus == nil {
continue
}
if !videoStartPTSFilled {
videoStartPTSFilled = true
videoStartPTS = tdata.pts
}
pts := tdata.pts - videoStartPTS
err := m.muxer.WriteH264(time.Now(), pts, tdata.nalus)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
} else if audioTrack != nil && data.getTrackID() == audioTrackID {
tdata := data.(*dataMPEG4Audio)
if tdata.aus == nil {
continue
}
if !audioStartPTSFilled {
audioStartPTSFilled = true
audioStartPTS = tdata.pts
}
pts := tdata.pts - audioStartPTS
for i, au := range tdata.aus {
err := m.muxer.WriteAAC(
time.Now(),
pts+time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/time.Duration(audioTrack.ClockRate()),
au)
if err != nil {
return fmt.Errorf("muxer error: %v", err)
}
}
}
}
}
func (m *hlsMuxer) handleRequest(req *hlsMuxerRequest) func() *hls.MuxerFileResponse { func (m *hlsMuxer) handleRequest(req *hlsMuxerRequest) func() *hls.MuxerFileResponse {
atomic.StoreInt64(m.lastRequestTime, time.Now().UnixNano()) atomic.StoreInt64(m.lastRequestTime, time.Now().UnixNano())
@ -558,7 +570,7 @@ func (m *hlsMuxer) apiHLSMuxersList(req hlsServerAPIMuxersListSubReq) {
} }
// onReaderData implements reader. // onReaderData implements reader.
func (m *hlsMuxer) onReaderData(data *data) { func (m *hlsMuxer) onReaderData(data data) {
m.ringBuffer.Push(data) m.ringBuffer.Push(data)
} }

19
internal/core/hls_source.go

@ -79,21 +79,26 @@ func (s *hlsSource) run(ctx context.Context) error {
} }
onVideoData := func(pts time.Duration, nalus [][]byte) { onVideoData := func(pts time.Duration, nalus [][]byte) {
stream.writeData(&data{ err := stream.writeData(&dataH264{
trackID: videoTrackID, trackID: videoTrackID,
ptsEqualsDTS: h264.IDRPresent(nalus), ptsEqualsDTS: h264.IDRPresent(nalus),
pts: pts, pts: pts,
h264NALUs: nalus, nalus: nalus,
}) })
if err != nil {
s.Log(logger.Warn, "%v", err)
}
} }
onAudioData := func(pts time.Duration, au []byte) { onAudioData := func(pts time.Duration, au []byte) {
stream.writeData(&data{ err := stream.writeData(&dataMPEG4Audio{
trackID: audioTrackID, trackID: audioTrackID,
ptsEqualsDTS: true, pts: pts,
pts: pts, aus: [][]byte{au},
mpeg4AudioAU: au,
}) })
if err != nil {
s.Log(logger.Warn, "%v", err)
}
} }
c, err := hls.NewClient( c, err := hls.NewClient(

26
internal/core/hls_source_test.go

@ -14,6 +14,7 @@ import (
"github.com/aler9/gortsplib/pkg/url" "github.com/aler9/gortsplib/pkg/url"
"github.com/asticode/go-astits" "github.com/asticode/go-astits"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/pion/rtp"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -136,11 +137,26 @@ func TestHLSSource(t *testing.T) {
c := gortsplib.Client{ c := gortsplib.Client{
OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) { OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) {
require.Equal(t, [][]byte{ require.Equal(t, &rtp.Packet{
{0x07, 0x01, 0x02, 0x03}, Header: rtp.Header{
{0x08}, Version: 2,
{0x05}, Marker: true,
}, ctx.H264NALUs) PayloadType: 96,
SequenceNumber: ctx.Packet.SequenceNumber,
Timestamp: ctx.Packet.Timestamp,
SSRC: ctx.Packet.SSRC,
CSRC: []uint32{},
},
Payload: []byte{
0x18,
0x00, 0x04,
0x07, 0x01, 0x02, 0x03, // SPS
0x00, 0x01,
0x08, // PPS
0x00, 0x01,
0x05, // ODR
},
}, ctx.Packet)
close(frameRecv) close(frameRecv)
}, },
} }

2
internal/core/reader.go

@ -3,6 +3,6 @@ package core
// reader is an entity that can read a stream. // reader is an entity that can read a stream.
type reader interface { type reader interface {
close() close()
onReaderData(*data) onReaderData(data)
apiReaderDescribe() interface{} apiReaderDescribe() interface{}
} }

7
internal/core/rpicamera_source.go

@ -59,12 +59,15 @@ func (s *rpiCameraSource) run(ctx context.Context) error {
stream = res.stream stream = res.stream
} }
stream.writeData(&data{ err := stream.writeData(&dataH264{
trackID: 0, trackID: 0,
ptsEqualsDTS: h264.IDRPresent(nalus), ptsEqualsDTS: h264.IDRPresent(nalus),
pts: dts, pts: dts,
h264NALUs: nalus, nalus: nalus,
}) })
if err != nil {
s.Log(logger.Warn, "%v", err)
}
} }
cam, err := rpicamera.New(s.params, onData) cam, err := rpicamera.New(s.params, onData)

101
internal/core/rtmp_conn.go

@ -14,7 +14,6 @@ import (
"github.com/aler9/gortsplib/pkg/h264" "github.com/aler9/gortsplib/pkg/h264"
"github.com/aler9/gortsplib/pkg/mpeg4audio" "github.com/aler9/gortsplib/pkg/mpeg4audio"
"github.com/aler9/gortsplib/pkg/ringbuffer" "github.com/aler9/gortsplib/pkg/ringbuffer"
"github.com/aler9/gortsplib/pkg/rtpmpeg4audio"
"github.com/notedit/rtmp/format/flv/flvio" "github.com/notedit/rtmp/format/flv/flvio"
"github.com/aler9/rtsp-simple-server/internal/conf" "github.com/aler9/rtsp-simple-server/internal/conf"
@ -258,7 +257,6 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
videoTrackID := -1 videoTrackID := -1
var audioTrack *gortsplib.TrackMPEG4Audio var audioTrack *gortsplib.TrackMPEG4Audio
audioTrackID := -1 audioTrackID := -1
var aacDecoder *rtpmpeg4audio.Decoder
for i, track := range res.stream.tracks() { for i, track := range res.stream.tracks() {
switch tt := track.(type) { switch tt := track.(type) {
@ -277,13 +275,6 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
audioTrack = tt audioTrack = tt
audioTrackID = i audioTrackID = i
aacDecoder = &rtpmpeg4audio.Decoder{
SampleRate: tt.Config.SampleRate,
SizeLength: tt.SizeLength,
IndexLength: tt.IndexLength,
IndexDeltaLength: tt.IndexDeltaLength,
}
aacDecoder.Init()
} }
} }
@ -336,7 +327,11 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
// disable read deadline // disable read deadline
c.nconn.SetReadDeadline(time.Time{}) c.nconn.SetReadDeadline(time.Time{})
var videoInitialPTS *time.Duration videoStartPTSFilled := false
var videoStartPTS time.Duration
audioStartPTSFilled := false
var audioStartPTS time.Duration
videoFirstIDRFound := false videoFirstIDRFound := false
var videoStartDTS time.Duration var videoStartDTS time.Duration
var videoDTSExtractor *h264.DTSExtractor var videoDTSExtractor *h264.DTSExtractor
@ -346,27 +341,25 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
if !ok { if !ok {
return fmt.Errorf("terminated") return fmt.Errorf("terminated")
} }
data := item.(*data) data := item.(data)
if videoTrack != nil && data.getTrackID() == videoTrackID {
tdata := data.(*dataH264)
if videoTrack != nil && data.trackID == videoTrackID { if tdata.nalus == nil {
if data.h264NALUs == nil {
continue continue
} }
// video is decoded in another routine, if !videoStartPTSFilled {
// while audio is decoded in this routine: videoStartPTSFilled = true
// we have to sync their PTS. videoStartPTS = tdata.pts
if videoInitialPTS == nil {
v := data.pts
videoInitialPTS = &v
} }
pts := tdata.pts - videoStartPTS
pts := data.pts - *videoInitialPTS
idrPresent := false idrPresent := false
nonIDRPresent := false nonIDRPresent := false
for _, nalu := range data.h264NALUs { for _, nalu := range tdata.nalus {
typ := h264.NALUType(nalu[0] & 0x1F) typ := h264.NALUType(nalu[0] & 0x1F)
switch typ { switch typ {
case h264.NALUTypeIDR: case h264.NALUTypeIDR:
@ -389,7 +382,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
videoDTSExtractor = h264.NewDTSExtractor() videoDTSExtractor = h264.NewDTSExtractor()
var err error var err error
dts, err = videoDTSExtractor.Extract(data.h264NALUs, pts) dts, err = videoDTSExtractor.Extract(tdata.nalus, pts)
if err != nil { if err != nil {
return err return err
} }
@ -403,7 +396,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
} }
var err error var err error
dts, err = videoDTSExtractor.Extract(data.h264NALUs, pts) dts, err = videoDTSExtractor.Extract(tdata.nalus, pts)
if err != nil { if err != nil {
return err return err
} }
@ -412,7 +405,7 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
pts -= videoStartDTS pts -= videoStartDTS
} }
avcc, err := h264.AVCCMarshal(data.h264NALUs) avcc, err := h264.AVCCMarshal(tdata.nalus)
if err != nil { if err != nil {
return err return err
} }
@ -430,25 +423,31 @@ func (c *rtmpConn) runRead(ctx context.Context, u *url.URL) error {
if err != nil { if err != nil {
return err return err
} }
} else if audioTrack != nil && data.trackID == audioTrackID { } else if audioTrack != nil && data.getTrackID() == audioTrackID {
aus, pts, err := aacDecoder.Decode(data.rtpPacket) tdata := data.(*dataMPEG4Audio)
if err != nil {
if err != rtpmpeg4audio.ErrMorePacketsNeeded { if tdata.aus == nil {
c.log(logger.Warn, "unable to decode audio track: %v", err)
}
continue continue
} }
if videoTrack != nil && !videoFirstIDRFound { if !audioStartPTSFilled {
continue audioStartPTSFilled = true
audioStartPTS = tdata.pts
} }
pts := tdata.pts - audioStartPTS
pts -= videoStartDTS if videoTrack != nil {
if pts < 0 { if !videoFirstIDRFound {
continue continue
}
pts -= videoStartDTS
if pts < 0 {
continue
}
} }
for i, au := range aus { for i, au := range tdata.aus {
c.nconn.SetWriteDeadline(time.Now().Add(time.Duration(c.writeTimeout))) c.nconn.SetWriteDeadline(time.Now().Add(time.Duration(c.writeTimeout)))
err := c.conn.WriteMessage(&message.MsgAudio{ err := c.conn.WriteMessage(&message.MsgAudio{
ChunkStreamID: message.MsgAudioChunkStreamID, ChunkStreamID: message.MsgAudioChunkStreamID,
@ -559,12 +558,15 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
conf.PPS, conf.PPS,
} }
rres.stream.writeData(&data{ err := rres.stream.writeData(&dataH264{
trackID: videoTrackID, trackID: videoTrackID,
ptsEqualsDTS: false, ptsEqualsDTS: false,
pts: tmsg.DTS + tmsg.PTSDelta, pts: tmsg.DTS + tmsg.PTSDelta,
h264NALUs: nalus, nalus: nalus,
}) })
if err != nil {
c.log(logger.Warn, "%v", err)
}
} else if tmsg.H264Type == flvio.AVC_NALU { } else if tmsg.H264Type == flvio.AVC_NALU {
if videoTrack == nil { if videoTrack == nil {
return fmt.Errorf("received an H264 packet, but track is not set up") return fmt.Errorf("received an H264 packet, but track is not set up")
@ -595,12 +597,15 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
} }
} }
rres.stream.writeData(&data{ err = rres.stream.writeData(&dataH264{
trackID: videoTrackID, trackID: videoTrackID,
ptsEqualsDTS: h264.IDRPresent(validNALUs), ptsEqualsDTS: h264.IDRPresent(validNALUs),
pts: tmsg.DTS + tmsg.PTSDelta, pts: tmsg.DTS + tmsg.PTSDelta,
h264NALUs: validNALUs, nalus: validNALUs,
}) })
if err != nil {
c.log(logger.Warn, "%v", err)
}
} }
case *message.MsgAudio: case *message.MsgAudio:
@ -609,12 +614,14 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
return fmt.Errorf("received an AAC packet, but track is not set up") return fmt.Errorf("received an AAC packet, but track is not set up")
} }
rres.stream.writeData(&data{ err := rres.stream.writeData(&dataMPEG4Audio{
trackID: audioTrackID, trackID: audioTrackID,
ptsEqualsDTS: true, pts: tmsg.DTS,
pts: tmsg.DTS, aus: [][]byte{tmsg.Payload},
mpeg4AudioAU: tmsg.Payload,
}) })
if err != nil {
c.log(logger.Warn, "%v", err)
}
} }
} }
} }
@ -667,7 +674,7 @@ func (c *rtmpConn) authenticate(
} }
// onReaderData implements reader. // onReaderData implements reader.
func (c *rtmpConn) onReaderData(data *data) { func (c *rtmpConn) onReaderData(data data) {
c.ringBuffer.Push(data) c.ringBuffer.Push(data)
} }

19
internal/core/rtmp_source.go

@ -170,12 +170,15 @@ func (s *rtmpSource) run(ctx context.Context) error {
return fmt.Errorf("unable to decode AVCC: %v", err) return fmt.Errorf("unable to decode AVCC: %v", err)
} }
res.stream.writeData(&data{ err = res.stream.writeData(&dataH264{
trackID: videoTrackID, trackID: videoTrackID,
ptsEqualsDTS: h264.IDRPresent(nalus), ptsEqualsDTS: h264.IDRPresent(nalus),
pts: tmsg.DTS + tmsg.PTSDelta, pts: tmsg.DTS + tmsg.PTSDelta,
h264NALUs: nalus, nalus: nalus,
}) })
if err != nil {
s.Log(logger.Warn, "%v", err)
}
} }
case *message.MsgAudio: case *message.MsgAudio:
@ -184,12 +187,14 @@ func (s *rtmpSource) run(ctx context.Context) error {
return fmt.Errorf("received an AAC packet, but track is not set up") return fmt.Errorf("received an AAC packet, but track is not set up")
} }
res.stream.writeData(&data{ err := res.stream.writeData(&dataMPEG4Audio{
trackID: audioTrackID, trackID: audioTrackID,
ptsEqualsDTS: true, pts: tmsg.DTS,
pts: tmsg.DTS, aus: [][]byte{tmsg.Payload},
mpeg4AudioAU: tmsg.Payload,
}) })
if err != nil {
s.Log(logger.Warn, "%v", err)
}
} }
} }
} }

31
internal/core/rtsp_session.go

@ -9,6 +9,7 @@ import (
"github.com/aler9/gortsplib" "github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base" "github.com/aler9/gortsplib/pkg/base"
"github.com/pion/rtp"
"github.com/aler9/rtsp-simple-server/internal/conf" "github.com/aler9/rtsp-simple-server/internal/conf"
"github.com/aler9/rtsp-simple-server/internal/externalcmd" "github.com/aler9/rtsp-simple-server/internal/externalcmd"
@ -342,7 +343,7 @@ func (s *rtspSession) onPause(ctx *gortsplib.ServerHandlerOnPauseCtx) (*base.Res
} }
// onReaderData implements reader. // onReaderData implements reader.
func (s *rtspSession) onReaderData(data *data) { func (s *rtspSession) onReaderData(data data) {
// packets are routed to the session by gortsplib.ServerStream. // packets are routed to the session by gortsplib.ServerStream.
} }
@ -378,21 +379,33 @@ func (s *rtspSession) apiSourceDescribe() interface{} {
// onPacketRTP is called by rtspServer. // onPacketRTP is called by rtspServer.
func (s *rtspSession) onPacketRTP(ctx *gortsplib.ServerHandlerOnPacketRTPCtx) { func (s *rtspSession) onPacketRTP(ctx *gortsplib.ServerHandlerOnPacketRTPCtx) {
if ctx.H264NALUs != nil { var err error
s.stream.writeData(&data{
switch s.announcedTracks[ctx.TrackID].(type) {
case *gortsplib.TrackH264:
err = s.stream.writeData(&dataH264{
trackID: ctx.TrackID, trackID: ctx.TrackID,
rtpPacket: ctx.Packet, rtpPackets: []*rtp.Packet{ctx.Packet},
ptsEqualsDTS: ctx.PTSEqualsDTS, ptsEqualsDTS: ctx.PTSEqualsDTS,
pts: ctx.H264PTS,
h264NALUs: ctx.H264NALUs,
}) })
} else {
s.stream.writeData(&data{ case *gortsplib.TrackMPEG4Audio:
err = s.stream.writeData(&dataMPEG4Audio{
trackID: ctx.TrackID,
rtpPackets: []*rtp.Packet{ctx.Packet},
})
default:
err = s.stream.writeData(&dataGeneric{
trackID: ctx.TrackID, trackID: ctx.TrackID,
rtpPacket: ctx.Packet, rtpPackets: []*rtp.Packet{ctx.Packet},
ptsEqualsDTS: ctx.PTSEqualsDTS, ptsEqualsDTS: ctx.PTSEqualsDTS,
}) })
} }
if err != nil {
s.log(logger.Warn, "%v", err)
}
} }
// onDecodeError is called by rtspServer. // onDecodeError is called by rtspServer.

29
internal/core/rtsp_source.go

@ -11,6 +11,7 @@ import (
"github.com/aler9/gortsplib" "github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/base" "github.com/aler9/gortsplib/pkg/base"
"github.com/pion/rtp"
"github.com/aler9/gortsplib/pkg/url" "github.com/aler9/gortsplib/pkg/url"
"github.com/aler9/rtsp-simple-server/internal/conf" "github.com/aler9/rtsp-simple-server/internal/conf"
@ -143,21 +144,33 @@ func (s *rtspSource) run(ctx context.Context) error {
}() }()
c.OnPacketRTP = func(ctx *gortsplib.ClientOnPacketRTPCtx) { c.OnPacketRTP = func(ctx *gortsplib.ClientOnPacketRTPCtx) {
if ctx.H264NALUs != nil { var err error
res.stream.writeData(&data{
switch tracks[ctx.TrackID].(type) {
case *gortsplib.TrackH264:
err = res.stream.writeData(&dataH264{
trackID: ctx.TrackID, trackID: ctx.TrackID,
rtpPacket: ctx.Packet, rtpPackets: []*rtp.Packet{ctx.Packet},
ptsEqualsDTS: ctx.PTSEqualsDTS, ptsEqualsDTS: ctx.PTSEqualsDTS,
pts: ctx.H264PTS,
h264NALUs: ctx.H264NALUs,
}) })
} else {
res.stream.writeData(&data{ case *gortsplib.TrackMPEG4Audio:
err = res.stream.writeData(&dataMPEG4Audio{
trackID: ctx.TrackID,
rtpPackets: []*rtp.Packet{ctx.Packet},
})
default:
err = res.stream.writeData(&dataGeneric{
trackID: ctx.TrackID, trackID: ctx.TrackID,
rtpPacket: ctx.Packet, rtpPackets: []*rtp.Packet{ctx.Packet},
ptsEqualsDTS: ctx.PTSEqualsDTS, ptsEqualsDTS: ctx.PTSEqualsDTS,
}) })
} }
if err != nil {
s.Log(logger.Warn, "%v", err)
}
} }
_, err = c.Play(nil) _, err = c.Play(nil)

353
internal/core/rtsp_source_test.go

@ -1,14 +1,19 @@
package core package core
import ( import (
"bytes"
"crypto/tls" "crypto/tls"
"net"
"os" "os"
"strings"
"testing" "testing"
"time" "time"
"github.com/aler9/gortsplib" "github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/auth" "github.com/aler9/gortsplib/pkg/auth"
"github.com/aler9/gortsplib/pkg/base" "github.com/aler9/gortsplib/pkg/base"
"github.com/aler9/gortsplib/pkg/conn"
"github.com/aler9/gortsplib/pkg/headers"
"github.com/aler9/gortsplib/pkg/rtph264" "github.com/aler9/gortsplib/pkg/rtph264"
"github.com/aler9/gortsplib/pkg/url" "github.com/aler9/gortsplib/pkg/url"
"github.com/pion/rtp" "github.com/pion/rtp"
@ -237,11 +242,9 @@ func TestRTSPSourceNoPassword(t *testing.T) {
} }
func TestRTSPSourceDynamicH264Params(t *testing.T) { func TestRTSPSourceDynamicH264Params(t *testing.T) {
track := &gortsplib.TrackH264{ stream := gortsplib.NewServerStream(gortsplib.Tracks{&gortsplib.TrackH264{
PayloadType: 96, PayloadType: 96,
} }})
stream := gortsplib.NewServerStream(gortsplib.Tracks{track})
defer stream.Close() defer stream.Close()
s := gortsplib.Server{ s := gortsplib.Server{
@ -340,3 +343,345 @@ func TestRTSPSourceDynamicH264Params(t *testing.T) {
require.Equal(t, []byte{8, 1}, h264Track.SafePPS()) require.Equal(t, []byte{8, 1}, h264Track.SafePPS())
}() }()
} }
func TestRTSPSourceRemovePadding(t *testing.T) {
stream := gortsplib.NewServerStream(gortsplib.Tracks{&gortsplib.TrackH264{
PayloadType: 96,
}})
defer stream.Close()
s := gortsplib.Server{
Handler: &testServer{
onDescribe: func(ctx *gortsplib.ServerHandlerOnDescribeCtx) (*base.Response, *gortsplib.ServerStream, error) {
return &base.Response{
StatusCode: base.StatusOK,
}, stream, nil
},
onSetup: func(ctx *gortsplib.ServerHandlerOnSetupCtx) (*base.Response, *gortsplib.ServerStream, error) {
return &base.Response{
StatusCode: base.StatusOK,
}, stream, nil
},
onPlay: func(ctx *gortsplib.ServerHandlerOnPlayCtx) (*base.Response, error) {
return &base.Response{
StatusCode: base.StatusOK,
}, nil
},
},
RTSPAddress: "127.0.0.1:8555",
}
err := s.Start()
require.NoError(t, err)
defer s.Wait()
defer s.Close()
p, ok := newInstance("rtmpDisable: yes\n" +
"hlsDisable: yes\n" +
"paths:\n" +
" proxied:\n" +
" source: rtsp://127.0.0.1:8555/teststream\n")
require.Equal(t, true, ok)
defer p.Close()
time.Sleep(1 * time.Second)
packetRecv := make(chan struct{})
c := gortsplib.Client{
OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) {
require.Equal(t, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 123,
Timestamp: 45343,
SSRC: 563423,
CSRC: []uint32{},
},
Payload: []byte{0x01, 0x02, 0x03, 0x04},
}, ctx.Packet)
close(packetRecv)
},
}
u, err := url.Parse("rtsp://127.0.0.1:8554/proxied")
require.NoError(t, err)
err = c.Start(u.Scheme, u.Host)
require.NoError(t, err)
defer c.Close()
tracks, baseURL, _, err := c.Describe(u)
require.NoError(t, err)
err = c.SetupAndPlay(tracks, baseURL)
require.NoError(t, err)
stream.WritePacketRTP(0, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 123,
Timestamp: 45343,
SSRC: 563423,
Padding: true,
},
Payload: []byte{0x01, 0x02, 0x03, 0x04},
PaddingSize: 20,
}, true)
<-packetRecv
}
func TestRTSPSourceOversizedPackets(t *testing.T) {
l, err := net.Listen("tcp", "127.0.0.1:8555")
require.NoError(t, err)
defer l.Close()
connected := make(chan struct{})
serverDone := make(chan struct{})
defer func() { <-serverDone }()
go func() {
defer close(serverDone)
nconn, err := l.Accept()
require.NoError(t, err)
defer nconn.Close()
conn := conn.NewConn(nconn)
req, err := conn.ReadRequest()
require.NoError(t, err)
require.Equal(t, base.Options, req.Method)
err = conn.WriteResponse(&base.Response{
StatusCode: base.StatusOK,
Header: base.Header{
"Public": base.HeaderValue{strings.Join([]string{
string(base.Describe),
string(base.Setup),
string(base.Play),
}, ", ")},
},
})
require.NoError(t, err)
req, err = conn.ReadRequest()
require.NoError(t, err)
require.Equal(t, base.Describe, req.Method)
tracks := gortsplib.Tracks{&gortsplib.TrackH264{
PayloadType: 96,
SPS: []byte{0x01, 0x02, 0x03, 0x04},
PPS: []byte{0x01, 0x02, 0x03, 0x04},
}}
err = conn.WriteResponse(&base.Response{
StatusCode: base.StatusOK,
Header: base.Header{
"Content-Type": base.HeaderValue{"application/sdp"},
},
Body: tracks.Marshal(false),
})
require.NoError(t, err)
req, err = conn.ReadRequest()
require.NoError(t, err)
require.Equal(t, base.Setup, req.Method)
var inTH headers.Transport
err = inTH.Unmarshal(req.Header["Transport"])
require.NoError(t, err)
err = conn.WriteResponse(&base.Response{
StatusCode: base.StatusOK,
Header: base.Header{
"Transport": headers.Transport{
Delivery: func() *headers.TransportDelivery {
v := headers.TransportDeliveryUnicast
return &v
}(),
Protocol: headers.TransportProtocolTCP,
InterleavedIDs: inTH.InterleavedIDs,
}.Marshal(),
},
})
require.NoError(t, err)
req, err = conn.ReadRequest()
require.NoError(t, err)
require.Equal(t, base.Play, req.Method)
err = conn.WriteResponse(&base.Response{
StatusCode: base.StatusOK,
})
require.NoError(t, err)
<-connected
byts, _ := rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 123,
Timestamp: 45343,
SSRC: 563423,
Padding: true,
},
Payload: []byte{0x01, 0x02, 0x03, 0x04},
}.Marshal()
err = conn.WriteInterleavedFrame(&base.InterleavedFrame{
Channel: 0,
Payload: byts,
}, make([]byte, 1024))
require.NoError(t, err)
byts, _ = rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 124,
Timestamp: 45343,
SSRC: 563423,
Padding: true,
},
Payload: bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 2000/4),
}.Marshal()
err = conn.WriteInterleavedFrame(&base.InterleavedFrame{
Channel: 0,
Payload: byts,
}, make([]byte, 2048))
require.NoError(t, err)
byts, _ = rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 125,
Timestamp: 45343,
SSRC: 563423,
Padding: true,
},
Payload: []byte{0x01, 0x02, 0x03, 0x04},
}.Marshal()
err = conn.WriteInterleavedFrame(&base.InterleavedFrame{
Channel: 0,
Payload: byts,
}, make([]byte, 1024))
require.NoError(t, err)
req, err = conn.ReadRequest()
require.NoError(t, err)
require.Equal(t, base.Teardown, req.Method)
err = conn.WriteResponse(&base.Response{
StatusCode: base.StatusOK,
})
require.NoError(t, err)
}()
p, ok := newInstance("rtmpDisable: yes\n" +
"hlsDisable: yes\n" +
"paths:\n" +
" proxied:\n" +
" source: rtsp://127.0.0.1:8555/teststream\n" +
" sourceProtocol: tcp\n")
require.Equal(t, true, ok)
defer p.Close()
time.Sleep(1 * time.Second)
packetRecv := make(chan struct{})
i := 0
c := gortsplib.Client{
OnPacketRTP: func(ctx *gortsplib.ClientOnPacketRTPCtx) {
switch i {
case 0:
require.Equal(t, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 123,
Timestamp: 45343,
SSRC: 563423,
CSRC: []uint32{},
},
Payload: []byte{0x01, 0x02, 0x03, 0x04},
}, ctx.Packet)
case 1:
require.Equal(t, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: false,
PayloadType: 96,
SequenceNumber: 124,
Timestamp: 45343,
SSRC: 563423,
CSRC: []uint32{},
},
Payload: append(
append([]byte{0x1c, 0x81, 0x02, 0x03, 0x04}, bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 363)...),
[]byte{0x01, 0x02, 0x03}...,
),
}, ctx.Packet)
case 2:
require.Equal(t, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 125,
Timestamp: 45343,
SSRC: 563423,
CSRC: []uint32{},
},
Payload: append(
[]byte{0x1c, 0x41, 0x04},
bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 135)...,
),
}, ctx.Packet)
case 3:
require.Equal(t, &rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 96,
SequenceNumber: 126,
Timestamp: 45343,
SSRC: 563423,
CSRC: []uint32{},
},
Payload: []byte{0x01, 0x02, 0x03, 0x04},
}, ctx.Packet)
close(packetRecv)
}
i++
},
}
u, err := url.Parse("rtsp://127.0.0.1:8554/proxied")
require.NoError(t, err)
err = c.Start(u.Scheme, u.Host)
require.NoError(t, err)
defer c.Close()
tracks, baseURL, _, err := c.Describe(u)
require.NoError(t, err)
err = c.SetupAndPlay(tracks, baseURL)
require.NoError(t, err)
close(connected)
<-packetRecv
}

29
internal/core/stream.go

@ -35,7 +35,7 @@ func (m *streamNonRTSPReadersMap) remove(r reader) {
delete(m.ma, r) delete(m.ma, r)
} }
func (m *streamNonRTSPReadersMap) writeData(data *data) { func (m *streamNonRTSPReadersMap) writeData(data data) {
m.mutex.RLock() m.mutex.RLock()
defer m.mutex.RUnlock() defer m.mutex.RUnlock()
@ -44,6 +44,12 @@ func (m *streamNonRTSPReadersMap) writeData(data *data) {
} }
} }
func (m *streamNonRTSPReadersMap) hasReaders() bool {
m.mutex.RLock()
defer m.mutex.RUnlock()
return len(m.ma) > 0
}
type stream struct { type stream struct {
nonRTSPReaders *streamNonRTSPReadersMap nonRTSPReaders *streamNonRTSPReadersMap
rtspStream *gortsplib.ServerStream rtspStream *gortsplib.ServerStream
@ -60,7 +66,7 @@ func newStream(tracks gortsplib.Tracks, generateRTPPackets bool) (*stream, error
for i, track := range s.rtspStream.Tracks() { for i, track := range s.rtspStream.Tracks() {
var err error var err error
s.streamTracks[i], err = newStreamTrack(track, generateRTPPackets, s.writeDataInner) s.streamTracks[i], err = newStreamTrack(track, generateRTPPackets)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -90,14 +96,19 @@ func (s *stream) readerRemove(r reader) {
} }
} }
func (s *stream) writeData(data *data) { func (s *stream) writeData(data data) error {
s.streamTracks[data.trackID].writeData(data) err := s.streamTracks[data.getTrackID()].onData(data, s.nonRTSPReaders.hasReaders())
} if err != nil {
return err
}
func (s *stream) writeDataInner(data *data) { // forward RTP packets to RTSP readers
// forward to RTSP readers for _, pkt := range data.getRTPPackets() {
s.rtspStream.WritePacketRTP(data.trackID, data.rtpPacket, data.ptsEqualsDTS) s.rtspStream.WritePacketRTP(data.getTrackID(), pkt, data.getPTSEqualsDTS())
}
// forward to non-RTSP readers // forward data to non-RTSP readers
s.nonRTSPReaders.writeData(data) s.nonRTSPReaders.writeData(data)
return nil
} }

10
internal/core/streamtrack.go

@ -7,21 +7,21 @@ import (
) )
type streamTrack interface { type streamTrack interface {
writeData(*data) onData(data, bool) error
} }
func newStreamTrack(track gortsplib.Track, generateRTPPackets bool, writeDataInner func(*data)) (streamTrack, error) { func newStreamTrack(track gortsplib.Track, generateRTPPackets bool) (streamTrack, error) {
switch ttrack := track.(type) { switch ttrack := track.(type) {
case *gortsplib.TrackH264: case *gortsplib.TrackH264:
return newStreamTrackH264(ttrack, generateRTPPackets, writeDataInner), nil return newStreamTrackH264(ttrack, generateRTPPackets), nil
case *gortsplib.TrackMPEG4Audio: case *gortsplib.TrackMPEG4Audio:
return newStreamTrackMPEG4Audio(ttrack, generateRTPPackets, writeDataInner), nil return newStreamTrackMPEG4Audio(ttrack, generateRTPPackets), nil
default: default:
if generateRTPPackets { if generateRTPPackets {
return nil, fmt.Errorf("we don't know how to generate RTP packets of track %+v", track) return nil, fmt.Errorf("we don't know how to generate RTP packets of track %+v", track)
} }
return newStreamTrackGeneric(track, writeDataInner), nil return newStreamTrackGeneric(), nil
} }
} }

32
internal/core/streamtrack_generic.go

@ -1,19 +1,33 @@
package core package core
import ( import (
"github.com/aler9/gortsplib" "fmt"
) )
type streamTrackGeneric struct { const (
writeDataInner func(*data) // 1500 (UDP MTU) - 20 (IP header) - 8 (UDP header)
maxPacketSize = 1472
)
type streamTrackGeneric struct{}
func newStreamTrackGeneric() *streamTrackGeneric {
return &streamTrackGeneric{}
} }
func newStreamTrackGeneric(track gortsplib.Track, writeDataInner func(*data)) *streamTrackGeneric { func (t *streamTrackGeneric) onData(dat data, hasNonRTSPReaders bool) error {
return &streamTrackGeneric{ tdata := dat.(*dataGeneric)
writeDataInner: writeDataInner,
pkt := tdata.rtpPackets[0]
// remove padding
pkt.Header.Padding = false
pkt.PaddingSize = 0
if pkt.MarshalSize() > maxPacketSize {
return fmt.Errorf("payload size (%d) is greater than maximum allowed (%d)",
pkt.MarshalSize(), maxPacketSize)
} }
}
func (t *streamTrackGeneric) writeData(data *data) { return nil
t.writeDataInner(data)
} }

168
internal/core/streamtrack_h264.go

@ -6,34 +6,97 @@ import (
"github.com/aler9/gortsplib" "github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/h264" "github.com/aler9/gortsplib/pkg/h264"
"github.com/aler9/gortsplib/pkg/rtph264" "github.com/aler9/gortsplib/pkg/rtph264"
"github.com/pion/rtp"
) )
func rtpH264ExtractSPSPPS(pkt *rtp.Packet) ([]byte, []byte) {
if len(pkt.Payload) == 0 {
return nil, nil
}
typ := h264.NALUType(pkt.Payload[0] & 0x1F)
switch typ {
case h264.NALUTypeSPS:
return pkt.Payload, nil
case h264.NALUTypePPS:
return nil, pkt.Payload
case 24: // STAP-A
payload := pkt.Payload[1:]
var sps []byte
var pps []byte
for len(payload) > 0 {
if len(payload) < 2 {
break
}
size := uint16(payload[0])<<8 | uint16(payload[1])
payload = payload[2:]
if size == 0 || int(size) > len(payload) {
break
}
nalu := payload[:size]
payload = payload[size:]
typ = h264.NALUType(nalu[0] & 0x1F)
switch typ {
case h264.NALUTypeSPS:
sps = nalu
case h264.NALUTypePPS:
pps = nalu
}
}
return sps, pps
default:
return nil, nil
}
}
type streamTrackH264 struct { type streamTrackH264 struct {
track *gortsplib.TrackH264 track *gortsplib.TrackH264
writeDataInner func(*data)
rtpEncoder *rtph264.Encoder encoder *rtph264.Encoder
decoder *rtph264.Decoder
} }
func newStreamTrackH264( func newStreamTrackH264(
track *gortsplib.TrackH264, track *gortsplib.TrackH264,
generateRTPPackets bool, generateRTPPackets bool,
writeDataInner func(*data),
) *streamTrackH264 { ) *streamTrackH264 {
t := &streamTrackH264{ t := &streamTrackH264{
track: track, track: track,
writeDataInner: writeDataInner,
} }
if generateRTPPackets { if generateRTPPackets {
t.rtpEncoder = &rtph264.Encoder{PayloadType: 96} t.encoder = &rtph264.Encoder{PayloadType: 96}
t.rtpEncoder.Init() t.encoder.Init()
} }
return t return t
} }
func (t *streamTrackH264) updateTrackParameters(nalus [][]byte) { func (t *streamTrackH264) updateTrackParametersFromRTPPacket(pkt *rtp.Packet) {
sps, pps := rtpH264ExtractSPSPPS(pkt)
if sps != nil && !bytes.Equal(sps, t.track.SafeSPS()) {
t.track.SafeSetSPS(sps)
}
if pps != nil && !bytes.Equal(pps, t.track.SafePPS()) {
t.track.SafeSetPPS(pps)
}
}
func (t *streamTrackH264) updateTrackParametersFromNALUs(nalus [][]byte) {
for _, nalu := range nalus { for _, nalu := range nalus {
typ := h264.NALUType(nalu[0] & 0x1F) typ := h264.NALUType(nalu[0] & 0x1F)
@ -105,40 +168,69 @@ func (t *streamTrackH264) remuxNALUs(nalus [][]byte) [][]byte {
return filteredNALUs return filteredNALUs
} }
func (t *streamTrackH264) generateRTPPackets(dat *data) { func (t *streamTrackH264) generateRTPPackets(tdata *dataH264) error {
pkts, err := t.rtpEncoder.Encode(dat.h264NALUs, dat.pts) pkts, err := t.encoder.Encode(tdata.nalus, tdata.pts)
if err != nil { if err != nil {
return return err
} }
lastPkt := len(pkts) - 1 tdata.rtpPackets = pkts
for i, pkt := range pkts { return nil
if i != lastPkt {
t.writeDataInner(&data{
trackID: dat.trackID,
rtpPacket: pkt,
})
} else {
t.writeDataInner(&data{
trackID: dat.trackID,
rtpPacket: pkt,
ptsEqualsDTS: dat.ptsEqualsDTS,
pts: dat.pts,
h264NALUs: dat.h264NALUs,
})
}
}
} }
func (t *streamTrackH264) writeData(dat *data) { func (t *streamTrackH264) onData(dat data, hasNonRTSPReaders bool) error {
if dat.h264NALUs != nil { tdata := dat.(*dataH264)
t.updateTrackParameters(dat.h264NALUs)
dat.h264NALUs = t.remuxNALUs(dat.h264NALUs) if tdata.rtpPackets != nil {
} pkt := tdata.rtpPackets[0]
t.updateTrackParametersFromRTPPacket(pkt)
if t.encoder == nil {
// remove padding
pkt.Header.Padding = false
pkt.PaddingSize = 0
// we need to re-encode since RTP packets exceed maximum size
if pkt.MarshalSize() > maxPacketSize {
v1 := pkt.SSRC
v2 := pkt.SequenceNumber
v3 := pkt.Timestamp
t.encoder = &rtph264.Encoder{
PayloadType: pkt.PayloadType,
SSRC: &v1,
InitialSequenceNumber: &v2,
InitialTimestamp: &v3,
}
t.encoder.Init()
}
}
// decode from RTP
if hasNonRTSPReaders || t.encoder != nil {
if t.decoder == nil {
t.decoder = &rtph264.Decoder{}
t.decoder.Init()
}
nalus, pts, err := t.decoder.Decode(pkt)
if err != nil {
return err
}
if dat.rtpPacket != nil { tdata.nalus = nalus
t.writeDataInner(dat) tdata.pts = pts
} else if dat.h264NALUs != nil {
t.generateRTPPackets(dat) tdata.nalus = t.remuxNALUs(tdata.nalus)
}
// route packet as is
if t.encoder == nil {
return nil
}
} else {
t.updateTrackParametersFromNALUs(tdata.nalus)
tdata.nalus = t.remuxNALUs(tdata.nalus)
} }
return t.generateRTPPackets(tdata)
} }

75
internal/core/streamtrack_mpeg4audio.go

@ -1,58 +1,89 @@
package core package core
import ( import (
"fmt"
"github.com/aler9/gortsplib" "github.com/aler9/gortsplib"
"github.com/aler9/gortsplib/pkg/rtpmpeg4audio" "github.com/aler9/gortsplib/pkg/rtpmpeg4audio"
) )
type streamTrackMPEG4Audio struct { type streamTrackMPEG4Audio struct {
writeDataInner func(*data) track *gortsplib.TrackMPEG4Audio
encoder *rtpmpeg4audio.Encoder
rtpEncoder *rtpmpeg4audio.Encoder decoder *rtpmpeg4audio.Decoder
} }
func newStreamTrackMPEG4Audio( func newStreamTrackMPEG4Audio(
track *gortsplib.TrackMPEG4Audio, track *gortsplib.TrackMPEG4Audio,
generateRTPPackets bool, generateRTPPackets bool,
writeDataInner func(*data),
) *streamTrackMPEG4Audio { ) *streamTrackMPEG4Audio {
t := &streamTrackMPEG4Audio{ t := &streamTrackMPEG4Audio{
writeDataInner: writeDataInner, track: track,
} }
if generateRTPPackets { if generateRTPPackets {
t.rtpEncoder = &rtpmpeg4audio.Encoder{ t.encoder = &rtpmpeg4audio.Encoder{
PayloadType: 96, PayloadType: 96,
SampleRate: track.ClockRate(), SampleRate: track.ClockRate(),
SizeLength: 13, SizeLength: 13,
IndexLength: 3, IndexLength: 3,
IndexDeltaLength: 3, IndexDeltaLength: 3,
} }
t.rtpEncoder.Init() t.encoder.Init()
} }
return t return t
} }
func (t *streamTrackMPEG4Audio) generateRTPPackets(dat *data) { func (t *streamTrackMPEG4Audio) generateRTPPackets(tdata *dataMPEG4Audio) error {
pkts, err := t.rtpEncoder.Encode([][]byte{dat.mpeg4AudioAU}, dat.pts) pkts, err := t.encoder.Encode(tdata.aus, tdata.pts)
if err != nil { if err != nil {
return return err
} }
for _, pkt := range pkts { tdata.rtpPackets = pkts
t.writeDataInner(&data{ return nil
trackID: dat.trackID,
rtpPacket: pkt,
ptsEqualsDTS: true,
})
}
} }
func (t *streamTrackMPEG4Audio) writeData(dat *data) { func (t *streamTrackMPEG4Audio) onData(dat data, hasNonRTSPReaders bool) error {
if dat.rtpPacket != nil { tdata := dat.(*dataMPEG4Audio)
t.writeDataInner(dat)
} else { if tdata.rtpPackets != nil {
t.generateRTPPackets(dat) pkt := tdata.rtpPackets[0]
// remove padding
pkt.Header.Padding = false
pkt.PaddingSize = 0
if pkt.MarshalSize() > maxPacketSize {
return fmt.Errorf("payload size (%d) is greater than maximum allowed (%d)",
pkt.MarshalSize(), maxPacketSize)
}
// decode from RTP
if hasNonRTSPReaders {
if t.decoder == nil {
t.decoder = &rtpmpeg4audio.Decoder{
SampleRate: t.track.Config.SampleRate,
SizeLength: t.track.SizeLength,
IndexLength: t.track.IndexLength,
IndexDeltaLength: t.track.IndexDeltaLength,
}
t.decoder.Init()
}
aus, pts, err := t.decoder.Decode(pkt)
if err != nil {
return err
}
tdata.aus = aus
tdata.pts = pts
}
// route packet as is
return nil
} }
return t.generateRTPPackets(tdata)
} }

8
internal/hls/muxer_variant_fmp4_segmenter.go

@ -213,7 +213,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(
} }
// put samples into a queue in order to // put samples into a queue in order to
// - allow to compute sample duration // - compute sample duration
// - check if next sample is IDR // - check if next sample is IDR
sample, m.nextVideoSample = m.nextVideoSample, sample sample, m.nextVideoSample = m.nextVideoSample, sample
if sample == nil { if sample == nil {
@ -290,6 +290,9 @@ func (m *muxerVariantFMP4Segmenter) writeAAC(now time.Time, dts time.Duration, a
} }
dts -= m.startDTS dts -= m.startDTS
if dts < 0 {
return nil
}
} }
sample := &augmentedAudioSample{ sample := &augmentedAudioSample{
@ -299,8 +302,7 @@ func (m *muxerVariantFMP4Segmenter) writeAAC(now time.Time, dts time.Duration, a
dts: dts, dts: dts,
} }
// put samples into a queue in order to // put samples into a queue in order to compute the sample duration
// allow to compute the sample duration
sample, m.nextAudioSample = m.nextAudioSample, sample sample, m.nextAudioSample = m.nextAudioSample, sample
if sample == nil { if sample == nil {
return nil return nil

Loading…
Cancel
Save