@ -17,294 +17,790 @@
@@ -17,294 +17,790 @@
along with qTox . If not , see < http : //www.gnu.org/licenses/>.
*/
# include <iostream>
# include "videoframe.h"
# include <QMutexLocker>
# include <QDebug>
# include <vpx/vpx_image.h>
extern " C " {
# include <libavcodec/avcodec.h>
extern " C " {
# include <libavutil/imgutils.h>
# include <libswscale/swscale.h>
}
# include "videoframe.h"
# include "camerasource.h"
/**
@ class VideoFrame
VideoFrame takes ownership of an AVFrame * and allows fast conversions to other formats
Ownership of all video frame buffers is kept by the VideoFrame , even after conversion
All references to the frame data become invalid when the VideoFrame is deleted
We try to avoid pixel format conversions as much as possible , at the cost of some memory
All methods are thread - safe . If provided freelistCallback will be called by the destructor ,
unless releaseFrame was called in between .
*/
* @ struct ToxYUVFrame
* @ brief A simple structure to represent a ToxYUV video frame ( corresponds to a frame encoded
* under format : AV_PIX_FMT_YUV420P [ FFmpeg ] or VPX_IMG_FMT_I420 [ WebM ] ) .
*
* This structure exists for convenience and code clarity when ferrying YUV420 frames from one
* source to another . The buffers pointed to by the struct should not be owned by the struct nor
* should they be freed from the struct , instead this struct functions only as a simple alias to a
* more complicated frame container like AVFrame .
*
* The creation of this structure was done to replace existing code which mis - used vpx_image
* structs when passing frame data to toxcore .
*
*
* @ class VideoFrame
* @ brief An ownernship and management class for AVFrames .
*
* VideoFrame takes ownership of an AVFrame * and allows fast conversions to other formats .
* Ownership of all video frame buffers is kept by the VideoFrame , even after conversion . All
* references to the frame data become invalid when the VideoFrame is deleted . We try to avoid
* pixel format conversions as much as possible , at the cost of some memory .
*
* Every function in this class is thread safe apart from concurrent construction and deletion of
* the object .
*
* This class uses the phrase " frame alignment " to specify the property that each frame ' s width is
* equal to it ' s maximum linesize . Note : this is NOT " data alignment " which specifies how allocated
* buffers are aligned in memory . Though internally the two are related , unless otherwise specified
* all instances of the term " alignment " exposed from public functions refer to frame alignment .
*
* Frame alignment is an important concept because ToxAV does not support frames with linesizes not
* directly equal to the width .
*
*
* @ var VideoFrame : : dataAlignment
* @ brief Data alignment parameter used to populate AVFrame buffers .
*
* This field is public in effort to standardize the data alignment parameter for all AVFrame
* allocations .
*
* It ' s currently set to 32 - byte alignment for AVX2 support .
*
*
* @ class FrameBufferKey
* @ brief A class representing a structure that stores frame properties to be used as the key
* value for a std : : unordered_map .
*/
// Initialize static fields
VideoFrame : : AtomicIDType VideoFrame : : frameIDs { 0 } ;
std : : unordered_map < VideoFrame : : IDType , QMutex > VideoFrame : : mutexMap { } ;
std : : unordered_map < VideoFrame : : IDType , std : : unordered_map < VideoFrame : : IDType , std : : weak_ptr < VideoFrame > > > VideoFrame : : refsMap { } ;
QReadWriteLock VideoFrame : : refsLock { } ;
VideoFrame : : VideoFrame ( AVFrame * frame , int w , int h , int fmt , std : : function < void ( ) > freelistCallback )
: freelistCallback { freelistCallback } ,
frameOther { nullptr } , frameYUV420 { nullptr } , frameRGB24 { nullptr } ,
width { w } , height { h } , pixFmt { fmt }
/**
* @ brief Constructs a new instance of a VideoFrame , sourced by a given AVFrame pointer .
*
* @ param sourceID the VideoSource ' s ID to track the frame under .
* @ param sourceFrame the source AVFrame pointer to use , must be valid .
* @ param dimensions the dimensions of the AVFrame , obtained from the AVFrame if not given .
* @ param pixFmt the pixel format of the AVFrame , obtained from the AVFrame if not given .
* @ param freeSourceFrame whether to free the source frame buffers or not .
*/
VideoFrame : : VideoFrame ( IDType sourceID , AVFrame * sourceFrame , QRect dimensions , int pixFmt , bool freeSourceFrame )
: frameID ( frameIDs + + ) ,
sourceID ( sourceID ) ,
sourceDimensions ( dimensions ) ,
sourceFrameKey ( getFrameKey ( dimensions . size ( ) , pixFmt , sourceFrame - > linesize [ 0 ] ) ) ,
freeSourceFrame ( freeSourceFrame )
{
// Silences pointless swscale warning spam
// See libswscale/utils.c:1153 @ 74f0bd3
frame - > color_range = AVCOL_RANGE_MPEG ;
if ( pixFmt = = AV_PIX_FMT_YUVJ420P )
pixFmt = AV_PIX_FMT_YUV420P ;
else if ( pixFmt = = AV_PIX_FMT_YUVJ411P )
pixFmt = AV_PIX_FMT_YUV411P ;
else if ( pixFmt = = AV_PIX_FMT_YUVJ422P )
pixFmt = AV_PIX_FMT_YUV422P ;
else if ( pixFmt = = AV_PIX_FMT_YUVJ444P )
pixFmt = AV_PIX_FMT_YUV444P ;
else if ( pixFmt = = AV_PIX_FMT_YUVJ440P )
pixFmt = AV_PIX_FMT_YUV440P ;
else
frame - > color_range = AVCOL_RANGE_UNSPECIFIED ;
if ( pixFmt = = AV_PIX_FMT_YUV420P ) {
frameYUV420 = frame ;
} else if ( pixFmt = = AV_PIX_FMT_RGB24 ) {
frameRGB24 = frame ;
} else {
frameOther = frame ;
// We override the pixel format in the case a deprecated one is used
switch ( pixFmt )
{
case AV_PIX_FMT_YUVJ420P :
{
sourcePixelFormat = AV_PIX_FMT_YUV420P ;
sourceFrame - > color_range = AVCOL_RANGE_MPEG ;
break ;
}
}
VideoFrame : : VideoFrame ( AVFrame * frame , std : : function < void ( ) > freelistCallback )
: VideoFrame { frame , frame - > width , frame - > height , frame - > format , freelistCallback }
{
case AV_PIX_FMT_YUVJ411P :
{
sourcePixelFormat = AV_PIX_FMT_YUV411P ;
sourceFrame - > color_range = AVCOL_RANGE_MPEG ;
break ;
}
case AV_PIX_FMT_YUVJ422P :
{
sourcePixelFormat = AV_PIX_FMT_YUV422P ;
sourceFrame - > color_range = AVCOL_RANGE_MPEG ;
break ;
}
case AV_PIX_FMT_YUVJ444P :
{
sourcePixelFormat = AV_PIX_FMT_YUV444P ;
sourceFrame - > color_range = AVCOL_RANGE_MPEG ;
break ;
}
case AV_PIX_FMT_YUVJ440P :
{
sourcePixelFormat = AV_PIX_FMT_YUV440P ;
sourceFrame - > color_range = AVCOL_RANGE_MPEG ;
break ;
}
default : {
sourcePixelFormat = pixFmt ;
sourceFrame - > color_range = AVCOL_RANGE_UNSPECIFIED ;
}
}
frameBuffer [ sourceFrameKey ] = sourceFrame ;
}
VideoFrame : : VideoFrame ( AVFrame * frame )
: VideoFrame { frame , frame - > width , frame - > height , frame - > format , nullptr }
VideoFrame : : VideoFrame ( IDType sourceID , AVFrame * sourceFrame , bool freeSourceFrame )
: VideoFrame ( sourceID , sourceFrame , QRect { 0 , 0 , sourceFrame - > width , sourceFrame - > height } , sourceFrame - > format , freeSourceFrame ) { }
/**
* @ brief Destructor for VideoFrame .
*/
VideoFrame : : ~ VideoFrame ( )
{
// Release frame
frameLock . lockForWrite ( ) ;
deleteFrameBuffer ( ) ;
frameLock . unlock ( ) ;
// Delete tracked reference
refsLock . lockForRead ( ) ;
if ( refsMap . count ( sourceID ) > 0 )
{
QMutex & sourceMutex = mutexMap [ sourceID ] ;
sourceMutex . lock ( ) ;
refsMap [ sourceID ] . erase ( frameID ) ;
sourceMutex . unlock ( ) ;
}
refsLock . unlock ( ) ;
}
/**
@ brief VideoFrame constructor . Disable copy .
@ note Use a shared_ptr if you need copies .
*/
VideoFrame : : ~ VideoFrame ( )
* @ brief Returns the validity of this VideoFrame .
*
* A VideoFrame is valid if it manages at least one AVFrame . A VideoFrame can be invalidated
* by calling releaseFrame ( ) on it .
*
* @ return true if the VideoFrame is valid , false otherwise .
*/
bool VideoFrame : : isValid ( )
{
if ( freelistCallback )
freelistCallback ( ) ;
frameLock . lockForRead ( ) ;
bool retValue = frameBuffer . size ( ) > 0 ;
frameLock . unlock ( ) ;
releaseFrameLockless ( ) ;
return retValue ;
}
/**
@ brief Converts the VideoFrame to a QImage that shares our internal video buffer .
@ param size Size of resulting image .
@ return Converted image to RGB24 color model .
*/
QImage VideoFrame : : toQImage ( QSize size )
* @ brief Causes the VideoFrame class to maintain an internal reference for the frame .
*
* The internal reference is managed via a std : : weak_ptr such that it doesn ' t inhibit
* destruction of the object once all external references are no longer reachable .
*
* @ return a std : : shared_ptr holding a reference to this frame .
*/
std : : shared_ptr < VideoFrame > VideoFrame : : trackFrame ( )
{
if ( ! convertToRGB24 ( size ) )
return QImage ( ) ;
// Add frame to tracked reference list
refsLock . lockForRead ( ) ;
if ( refsMap . count ( sourceID ) = = 0 )
{
// We need to add a new source to our reference map, obtain write lock
refsLock . unlock ( ) ;
refsLock . lockForWrite ( ) ;
}
QMutex & sourceMutex = mutexMap [ sourceID ] ;
sourceMutex . lock ( ) ;
std : : shared_ptr < VideoFrame > ret { this } ;
refsMap [ sourceID ] [ frameID ] = ret ;
QMutexLocker locker ( & biglock ) ;
sourceMutex . unlock ( ) ;
refsLock . unlock ( ) ;
return QImage ( * frameRGB24 - > data , frameRGB24 - > width , frameRGB24 - > height , * frameRGB24 - > linesize , QImage : : Format_RGB888 ) ;
return ret ;
}
/**
@ brief Converts the VideoFrame to a vpx_image_t .
Converts the VideoFrame to a vpx_image_t that shares our internal video buffer .
@ return Converted image to vpx_image format .
*/
vpx_image * VideoFrame : : toVpxImage ( )
* @ brief Untracks all the frames for the given VideoSource , releasing them if specified .
*
* This function causes all internally tracked frames for the given VideoSource to be dropped .
* If the releaseFrames option is set to true , the frames are sequentially released on the
* caller ' s thread in an unspecified order .
*
* @ param sourceID the ID of the VideoSource to untrack frames from .
* @ param releaseFrames true to release the frames as necessary , false otherwise . Defaults to
* false .
*/
void VideoFrame : : untrackFrames ( const VideoFrame : : IDType & sourceID , bool releaseFrames )
{
vpx_image * img = vpx_img_alloc ( nullptr , VPX_IMG_FMT_I420 , width , height , 0 ) ;
refsLock . lockForWrite ( ) ;
if ( refsMap . count ( sourceID ) = = 0 )
{
// No tracking reference exists for source, simply return
refsLock . unlock ( ) ;
if ( ! convertToYUV420 ( ) )
return img ;
return ;
}
for ( int i = 0 ; i < 3 ; i + + )
if ( releaseFrames )
{
int dstStride = img - > stride [ i ] ;
int srcStride = frameYUV420 - > linesize [ i ] ;
int minStride = std : : min ( dstStride , srcStride ) ;
int size = ( i = = 0 ) ? img - > d_h : img - > d_h / 2 ;
QMutex & sourceMutex = mutexMap [ sourceID ] ;
sourceMutex . lock ( ) ;
for ( int j = 0 ; j < size ; j + + )
for ( auto & frameIterator : refsMap [ sourceID ] )
{
uint8_t * dst = img - > planes [ i ] + dstStride * j ;
uint8_t * src = frameYUV420 - > data [ i ] + srcStride * j ;
memcpy ( dst , src , minStride ) ;
std : : shared_ptr < VideoFrame > frame = frameIterator . second . lock ( ) ;
if ( frame )
{
frame - > releaseFrame ( ) ;
}
}
sourceMutex . unlock ( ) ;
}
return img ;
refsMap [ sourceID ] . clear ( ) ;
mutexMap . erase ( sourceID ) ;
refsMap . erase ( sourceID ) ;
refsLock . unlock ( ) ;
}
bool VideoFrame : : convertToRGB24 ( QSize size )
/**
* @ brief Releases all frames managed by this VideoFrame and invalidates it .
*/
void VideoFrame : : releaseFrame ( )
{
QMutexLocker locker ( & biglock ) ;
frameLock . lockForWrite ( ) ;
AVFrame * sourceFrame ;
if ( frameOther )
{
sourceFrame = frameOther ;
}
else if ( frameYUV420 )
deleteFrameBuffer ( ) ;
frameLock . unlock ( ) ;
}
/**
* @ brief Retrieves an AVFrame derived from the source based on the given parameters .
*
* If a given frame does not exist , this function will perform appropriate conversions to
* return a frame that fulfills the given parameters .
*
* @ param frameSize the dimensions of the frame to get . Defaults to source frame size if frameSize
* is invalid .
* @ param pixelFormat the desired pixel format of the frame .
* @ param requireAligned true if the returned frame must be frame aligned , false if not .
* @ return a pointer to a AVFrame with the given parameters or nullptr if the VideoFrame is no
* longer valid .
*/
const AVFrame * VideoFrame : : getAVFrame ( QSize frameSize , const int pixelFormat , const bool requireAligned )
{
if ( ! frameSize . isValid ( ) )
{
sourceFrame = frameYUV420 ;
frameSize = sourceDimensions . size ( ) ;
}
else
// Since we are retrieving the AVFrame* directly, we merely need to pass the arguement through
const std : : function < AVFrame * ( AVFrame * const ) > converter = [ ] ( AVFrame * const frame )
{
qWarning ( ) < < " None of the frames are valid! Did someone release us? " ;
return false ;
}
//std::cout << "converting to RGB24" << std::endl;
return frame ;
} ;
// We need an explicit null pointer holding object to pass to toGenericObject()
AVFrame * nullPointer = nullptr ;
if ( size . isEmpty ( ) )
// Returns std::nullptr case of invalid generation
return toGenericObject ( frameSize , pixelFormat , requireAligned , converter , nullPointer ) ;
}
/**
* @ brief Converts this VideoFrame to a QImage that shares this VideoFrame ' s buffer .
*
* The VideoFrame will be scaled into the RGB24 pixel format along with the given
* dimension .
*
* @ param frameSize the given frame size of QImage to generate . Defaults to source frame size if
* frameSize is invalid .
* @ return a QImage that represents this VideoFrame , sharing it ' s buffers or a null image if
* this VideoFrame is no longer valid .
*/
QImage VideoFrame : : toQImage ( QSize frameSize )
{
if ( ! frameSize . isValid ( ) )
{
size . setWidth ( sourceFrame - > width ) ;
size . setHeight ( sourceFrame - > height ) ;
frameSize = sourceDimensions . size ( ) ;
}
if ( frameRGB24 )
// Converter function (constructs QImage out of AVFrame*)
const std : : function < QImage ( AVFrame * const ) > converter = [ & ] ( AVFrame * const frame )
{
if ( frameRGB24 - > width = = size . width ( ) & & frameRGB24 - > height = = size . height ( ) )
return true ;
return QImage { * ( frame - > data ) , frameS ize. width ( ) , frameSize . height ( ) , * ( frame - > linesize ) , QImage : : Format_RGB888 } ;
} ;
av_free ( frameRGB24 - > opaque ) ;
av_frame_unref ( frameRGB24 ) ;
av_frame_free ( & frameRGB24 ) ;
}
// Returns an empty constructed QImage in case of invalid generation
return toGenericObject ( frameSize , AV_PIX_FMT_RGB24 , false , converter , QImage { } ) ;
}
frameRGB24 = av_frame_alloc ( ) ;
if ( ! frameRGB24 )
/**
* @ brief Converts this VideoFrame to a ToxAVFrame that shares this VideoFrame ' s buffer .
*
* The given ToxAVFrame will be frame aligned under a pixel format of planar YUV with a chroma
* subsampling format of 4 : 2 : 0 ( i . e . AV_PIX_FMT_YUV420P ) .
*
* @ param frameSize the given frame size of ToxAVFrame to generate . Defaults to source frame size
* if frameSize is invalid .
* @ return a ToxAVFrame structure that represents this VideoFrame , sharing it ' s buffers or an
* empty structure if this VideoFrame is no longer valid .
*/
ToxYUVFrame VideoFrame : : toToxYUVFrame ( QSize frameSize )
{
if ( ! frameSize . isValid ( ) )
{
qCritical ( ) < < " av_frame_alloc failed " ;
return false ;
frameSize = sourceDimensions . size ( ) ;
}
int imgBufferSize = av_image_get_buffer_size ( AV_PIX_FMT_RGB24 , size . width ( ) , size . height ( ) , 1 ) ;
uint8_t * buf = ( uint8_t * ) av_malloc ( imgBufferSize ) ;
if ( ! buf )
// Converter function (constructs ToxAVFrame out of AVFrame*)
const std : : function < ToxYUVFrame ( AVFrame * const ) > converter = [ & ] ( AVFrame * const frame )
{
qCritical ( ) < < " av_malloc failed " ;
av_frame_free ( & frameRGB24 ) ;
return false ;
}
frameRGB24 - > opaque = buf ;
ToxYUVFrame ret
{
static_cast < std : : uint16_t > ( frameSize . width ( ) ) ,
static_cast < std : : uint16_t > ( frameSize . height ( ) ) ,
frame - > data [ 0 ] , frame - > data [ 1 ] , frame - > data [ 2 ]
} ;
uint8_t * * data = frameRGB24 - > data ;
int * linesize = frameRGB24 - > linesize ;
av_image_fill_arrays ( data , linesize , buf , AV_PIX_FMT_RGB24 , size . width ( ) , size . height ( ) , 1 ) ;
frameRGB24 - > width = size . width ( ) ;
frameRGB24 - > height = size . height ( ) ;
return ret ;
} ;
// Bilinear is better for shrinking, bicubic better for upscaling
int resizeAlgo = size . width ( ) < = width ? SWS_BILINEAR : SWS_BICUBIC ;
SwsContext * swsCtx = sws_getContext ( width , height , ( AVPixelFormat ) pixFmt ,
size . width ( ) , size . height ( ) , AV_PIX_FMT_RGB24 ,
resizeAlgo , nullptr , nullptr , nullptr ) ;
sws_scale ( swsCtx , ( uint8_t const * const * ) sourceFrame - > data ,
sourceFrame - > linesize , 0 , height ,
frameRGB24 - > data , frameRGB24 - > linesize ) ;
sws_freeContext ( swsCtx ) ;
return toGenericObject ( frameSize , AV_PIX_FMT_YUV420P , true , converter , ToxYUVFrame { 0 , 0 , nullptr , nullptr , nullptr } ) ;
}
/**
* @ brief Returns the ID for the given frame .
*
* Frame IDs are globally unique ( with respect to the running instance ) .
*
* @ return an integer representing the ID of this frame .
*/
VideoFrame : : IDType VideoFrame : : getFrameID ( ) const
{
return frameID ;
}
/**
* @ brief Returns the ID for the VideoSource which created this frame .
*
* @ return an integer representing the ID of the VideoSource which created this frame .
*/
VideoFrame : : IDType VideoFrame : : getSourceID ( ) const
{
return sourceID ;
}
/**
* @ brief Retrieves a copy of the source VideoFrame ' s dimensions .
*
* @ return QRect copy representing the source VideoFrame ' s dimensions .
*/
QRect VideoFrame : : getSourceDimensions ( ) const
{
return sourceDimensions ;
}
/**
* @ brief Retrieves a copy of the source VideoFormat ' s pixel format .
*
* @ return integer copy representing the source VideoFrame ' s pixel format .
*/
int VideoFrame : : getSourcePixelFormat ( ) const
{
return sourcePixelFormat ;
}
/**
* @ brief Constructs a new FrameBufferKey with the given attributes .
*
* @ param width the width of the frame .
* @ param height the height of the frame .
* @ param pixFmt the pixel format of the frame .
* @ param lineAligned whether the linesize matches the width of the image .
*/
VideoFrame : : FrameBufferKey : : FrameBufferKey ( const int pixFmt , const int width , const int height , const bool lineAligned )
: frameWidth ( width ) ,
frameHeight ( height ) ,
pixelFormat ( pixFmt ) ,
linesizeAligned ( lineAligned ) { }
/**
* @ brief Comparison operator for FrameBufferKey .
*
* @ param other instance to compare against .
* @ return true if instances are equivilent , false otherwise .
*/
bool VideoFrame : : FrameBufferKey : : operator = = ( const FrameBufferKey & other ) const
{
return pixelFormat = = other . pixelFormat & &
frameWidth = = other . frameWidth & &
frameHeight = = other . frameHeight & &
linesizeAligned = = other . linesizeAligned ;
}
/**
* @ brief Not equal to operator for FrameBufferKey .
*
* @ param other instance to compare against
* @ return true if instances are not equivilent , false otherwise .
*/
bool VideoFrame : : FrameBufferKey : : operator ! = ( const FrameBufferKey & other ) const
{
return ! operator = = ( other ) ;
}
/**
* @ brief Hash function for FrameBufferKey .
*
* This function computes a hash value for use with std : : unordered_map .
*
* @ param key the given instance to compute hash value of .
* @ return the hash of the given instance .
*/
size_t VideoFrame : : FrameBufferKey : : hash ( const FrameBufferKey & key )
{
std : : hash < int > intHasher ;
std : : hash < bool > boolHasher ;
// Use java-style hash function to combine fields
// See: https://en.wikipedia.org/wiki/Java_hashCode%28%29#hashCode.28.29_in_general
size_t ret = 47 ;
ret = 37 * ret + intHasher ( key . frameWidth ) ;
ret = 37 * ret + intHasher ( key . frameHeight ) ;
ret = 37 * ret + intHasher ( key . pixelFormat ) ;
ret = 37 * ret + boolHasher ( key . linesizeAligned ) ;
return true ;
return ret ;
}
bool VideoFrame : : convertToYUV420 ( )
/**
* @ brief Generates a key object based on given parameters .
*
* @ param frameSize the given size of the frame .
* @ param pixFmt the pixel format of the frame .
* @ param linesize the maximum linesize of the frame , may be larger than the width .
* @ return a FrameBufferKey object representing the key for the frameBuffer map .
*/
VideoFrame : : FrameBufferKey VideoFrame : : getFrameKey ( const QSize & frameSize , const int pixFmt , const int linesize )
{
QMutexLocker locker ( & biglock ) ;
return getFrameKey ( frameSize , pixFmt , frameSize . width ( ) = = linesize ) ;
}
if ( frameYUV420 )
return true ;
/**
* @ brief Generates a key object based on given parameters .
*
* @ param frameSize the given size of the frame .
* @ param pixFmt the pixel format of the frame .
* @ param frameAligned true if the frame is aligned , false otherwise .
* @ return a FrameBufferKey object representing the key for the frameBuffer map .
*/
VideoFrame : : FrameBufferKey VideoFrame : : getFrameKey ( const QSize & frameSize , const int pixFmt , const bool frameAligned )
{
return { frameSize . width ( ) , frameSize . height ( ) , pixFmt , frameAligned } ;
}
AVFrame * sourceFrame ;
if ( frameOther )
/**
* @ brief Retrieves an AVFrame derived from the source based on the given parameters without
* obtaining a lock .
*
* This function is not thread - safe and must be called from a thread - safe context .
*
* Note : this function differs from getAVFrame ( ) in that it returns a nullptr if no frame was
* found .
*
* @ param dimensions the dimensions of the frame , must be valid .
* @ param pixelFormat the desired pixel format of the frame .
* @ param requireAligned true if the frame must be frame aligned , false otherwise .
* @ return a pointer to a AVFrame with the given parameters or nullptr if no such frame was
* found .
*/
AVFrame * VideoFrame : : retrieveAVFrame ( const QSize & dimensions , const int pixelFormat , const bool requireAligned )
{
if ( ! requireAligned )
{
sourceFrame = frameOther ;
/*
* We attempt to obtain a unaligned frame first because an unaligned linesize corresponds
* to a data aligned frame .
*/
FrameBufferKey frameKey = getFrameKey ( dimensions , pixelFormat , false ) ;
if ( frameBuffer . count ( frameKey ) > 0 )
{
return frameBuffer [ frameKey ] ;
}
}
else if ( frameRGB24 )
FrameBufferKey frameKey = getFrameKey ( dimensions , pixelFormat , true ) ;
if ( frameBuffer . count ( frameKey ) > 0 )
{
sourceFrame = frameRGB24 ;
return frameBuffer [ frameKey ] ;
}
else
{
qCritical ( ) < < " None of the frames are valid! Did someone release us? " ;
return false ;
return nullptr ;
}
}
/**
* @ brief Generates an AVFrame based on the given specifications .
*
* This function is not thread - safe and must be called from a thread - safe context .
*
* @ param dimensions the required dimensions for the frame , must be valid .
* @ param pixelFormat the required pixel format for the frame .
* @ param requireAligned true if the generated frame needs to be frame aligned , false otherwise .
* @ return an AVFrame with the given specifications .
*/
AVFrame * VideoFrame : : generateAVFrame ( const QSize & dimensions , const int pixelFormat , const bool requireAligned )
{
AVFrame * ret = av_frame_alloc ( ) ;
if ( ! ret ) {
return nullptr ;
}
//std::cout << "converting to YUV420" << std::endl;
frameYUV420 = av_frame_alloc ( ) ;
if ( ! frameYUV420 )
// Populate AVFrame fields
ret - > width = dimensions . width ( ) ;
ret - > height = dimensions . height ( ) ;
ret - > format = pixelFormat ;
/*
* We generate a frame under data alignment only if the dimensions allow us to be frame aligned
* or if the caller doesn ' t require frame alignment
*/
int bufSize ;
if ( ! requireAligned | | ( dimensions . width ( ) % 8 = = 0 & & dimensions . height ( ) % 8 = = 0 ) )
{
bufSize = av_image_alloc ( ret - > data , ret - > linesize ,
dimensions . width ( ) , dimensions . height ( ) ,
static_cast < AVPixelFormat > ( pixelFormat ) , dataAlignment ) ;
}
else
{
qCritical ( ) < < " av_frame_alloc failed " ;
return false ;
bufSize = av_image_alloc ( ret - > data , ret - > linesize ,
dimensions . width ( ) , dimensions . height ( ) ,
static_cast < AVPixelFormat > ( pixelFormat ) , 1 ) ;
}
int imgBufferSize = av_image_get_buffer_size ( AV_PIX_FMT_RGB24 , width , height , 1 ) ;
uint8_t * buf = ( uint8_t * ) av_malloc ( imgBufferSize ) ;
if ( ! buf )
if ( bufSize < 0 )
{
qCritical ( ) < < " av_malloc failed " ;
av_frame_free ( & frameYUV420 ) ;
return false ;
av_frame_free ( & ret ) ;
return nullptr ;
}
// Bilinear is better for shrinking, bicubic better for upscaling
int resizeAlgo = sourceDimensions . width ( ) > dimensions . width ( ) ? SWS_BILINEAR : SWS_BICUBIC ;
SwsContext * swsCtx = sws_getContext ( sourceDimensions . width ( ) , sourceDimensions . height ( ) ,
static_cast < AVPixelFormat > ( sourcePixelFormat ) ,
dimensions . width ( ) , dimensions . height ( ) ,
static_cast < AVPixelFormat > ( pixelFormat ) ,
resizeAlgo , nullptr , nullptr , nullptr ) ;
if ( ! swsCtx ) {
av_freep ( & ret - > data [ 0 ] ) ;
av_frame_unref ( ret ) ;
av_frame_free ( & ret ) ;
return nullptr ;
}
frameYUV420 - > opaque = buf ;
uint8_t * * data = frameYUV420 - > data ;
int * linesize = frameYUV420 - > linesize ;
av_image_fill_arrays ( data , linesize , buf , AV_PIX_FMT_YUV420P , width , height , 1 ) ;
SwsContext * swsCtx = sws_getContext ( width , height , ( AVPixelFormat ) pixFmt ,
width , height , AV_PIX_FMT_YUV420P ,
SWS_BILINEAR , nullptr , nullptr , nullptr ) ;
sws_scale ( swsCtx , ( uint8_t const * const * ) sourceFrame - > data ,
sourceFrame - > linesize , 0 , height ,
frameYUV420 - > data , frameYUV420 - > linesize ) ;
AVFrame * source = frameBuffer [ sourceFrameKey ] ;
sws_scale ( swsCtx , source - > data , source - > linesize , 0 , sourceDimensions . height ( ) , ret - > data , ret - > linesize ) ;
sws_freeContext ( swsCtx ) ;
return true ;
return ret ;
}
/**
@ brief Frees all frame memory .
Frees all internal buffers and frame data , removes the freelistCallback
This makes all converted objects that shares our internal buffers invalid .
*/
void VideoFrame : : releaseFrame ( )
* @ brief Stores a given AVFrame within the frameBuffer map .
*
* As protection against duplicate frames , the storage mechanism will only allow one frame of a
* given type to exist in the frame buffer . Should the given frame type already exist in the frame
* buffer , the given frame will be freed and have it ' s buffers invalidated . In order to ensure
* correct operation , always replace the frame pointer with the one returned by this function .
*
* As an example :
* @ code { . cpp }
* AVFrame * frame = // create AVFrame...
*
* frame = storeAVFrame ( frame , dimensions , pixelFormat ) ;
* @ endcode
*
* This function is not thread - safe and must be called from a thread - safe context .
*
* @ param frame the given frame to store .
* @ param dimensions the dimensions of the frame , must be valid .
* @ param pixelFormat the pixel format of the frame .
* @ return The given AVFrame * or a pre - existing AVFrame * that already exists in the frameBuffer .
*/
AVFrame * VideoFrame : : storeAVFrame ( AVFrame * frame , const QSize & dimensions , const int pixelFormat )
{
QMutexLocker locker ( & biglock ) ;
freelistCallback = nullptr ;
releaseFrameLockless ( ) ;
FrameBufferKey frameKey = getFrameKey ( dimensions , pixelFormat , frame - > linesize [ 0 ] ) ;
// We check the prescence of the frame in case of double-computation
if ( frameBuffer . count ( frameKey ) > 0 )
{
AVFrame * old_ret = frameBuffer [ frameKey ] ;
// Free new frame
av_freep ( & frame - > data [ 0 ] ) ;
av_frame_unref ( frame ) ;
av_frame_free ( & frame ) ;
return old_ret ;
}
else
{
frameBuffer [ frameKey ] = frame ;
return frame ;
}
}
void VideoFrame : : releaseFrameLockless ( )
/**
* @ brief Releases all frames within the frame buffer .
*
* This function is not thread - safe and must be called from a thread - safe context .
*/
void VideoFrame : : deleteFrameBuffer ( )
{
if ( frameOther )
// An empty framebuffer represents a frame that's already been freed
if ( frameBuffer . empty ( ) ) {
return ;
}
for ( const auto & frameIterator : frameBuffer )
{
av_free ( frameOther - > opaque ) ;
av_frame_unref ( frameOther ) ;
av_frame_free ( & frameOther ) ;
AVFrame * frame = frameIterator . second ;
// Treat source frame and derived frames separately
if ( sourceFrameKey = = frameIterator . first )
{
if ( freeSourceFrame )
{
av_freep ( & frame - > data [ 0 ] ) ;
}
av_frame_unref ( frame ) ;
av_frame_free ( & frame ) ;
}
else
{
av_freep ( & frame - > data [ 0 ] ) ;
av_frame_unref ( frame ) ;
av_frame_free ( & frame ) ;
}
}
if ( frameYUV420 )
frameBuffer . clear ( ) ;
}
/**
* @ brief Converts this VideoFrame to a generic type T based on the given parameters and
* supplied converter functions .
*
* This function is used internally to create various toXObject functions that all follow the
* same generation pattern ( where XObject is some existing type like QImage ) .
*
* In order to create such a type , a object constructor function is required that takes the
* generated AVFrame object and creates type T out of it . This function additionally requires
* a null object of type T that represents an invalid / null object for when the generation
* process fails ( e . g . when the VideoFrame is no longer valid ) .
*
* @ param dimensions the dimensions of the frame , must be valid .
* @ param pixelFormat the pixel format of the frame .
* @ param requireAligned true if the generated frame needs to be frame aligned , false otherwise .
* @ param objectConstructor a std : : function that takes the generated AVFrame and converts it
* to an object of type T .
* @ param nullObject an object of type T that represents the null / invalid object to be used
* when the generation process fails .
*/
template < typename T >
T VideoFrame : : toGenericObject ( const QSize & dimensions , const int pixelFormat , const bool requireAligned ,
const std : : function < T ( AVFrame * const ) > objectConstructor , const T & nullObject )
{
frameLock . lockForRead ( ) ;
// We return nullObject if the VideoFrame is no longer valid
if ( frameBuffer . size ( ) = = 0 )
{
av_free ( frameYUV420 - > opaque ) ;
av_frame_unref ( frameYUV420 ) ;
av_frame_free ( & frameYUV420 ) ;
frameLock . unlock ( ) ;
return nullObject ;
}
if ( frameRGB24 )
AVFrame * frame = retrieveAVFrame ( dimensions , static_cast < int > ( pixelFormat ) , requireAligned ) ;
if ( frame )
{
av_free ( frameRGB24 - > opaque ) ;
av_frame_unref ( frameRGB24 ) ;
av_frame_free ( & frameRGB24 ) ;
T ret = objectConstructor ( frame ) ;
frameLock . unlock ( ) ;
return ret ;
}
// VideoFrame does not contain an AVFrame to spec, generate one here
frame = generateAVFrame ( dimensions , static_cast < int > ( pixelFormat ) , requireAligned ) ;
/*
* We need to " upgrade " the lock to a write lock so we can update our frameBuffer map .
*
* It doesn ' t matter if another thread obtains the write lock before we finish since it is
* likely writing to somewhere else . Worst - case scenario , we merely perform the generation
* process twice , and discard the old result .
*/
frameLock . unlock ( ) ;
frameLock . lockForWrite ( ) ;
frame = storeAVFrame ( frame , dimensions , static_cast < int > ( pixelFormat ) ) ;
T ret = objectConstructor ( frame ) ;
frameLock . unlock ( ) ;
return ret ;
}
// Explicitly specialize VideoFrame::toGenericObject() function
template QImage VideoFrame : : toGenericObject < QImage > ( const QSize & dimensions , const int pixelFormat , const bool requireAligned ,
const std : : function < QImage ( AVFrame * const ) > objectConstructor , const QImage & nullObject ) ;
template ToxYUVFrame VideoFrame : : toGenericObject < ToxYUVFrame > ( const QSize & dimensions , const int pixelFormat , const bool requireAligned ,
const std : : function < ToxYUVFrame ( AVFrame * const ) > objectConstructor , const ToxYUVFrame & nullObject ) ;
/**
@ brief Return the size of the original frame
@ return The size of the original frame
*/
QSize VideoFrame : : getSize ( )
* @ brief Returns whether the given ToxYUVFrame represents a valid frame or not .
*
* Valid frames are frames in which both width and height are greater than zero .
*
* @ return true if the frame is valid , false otherwise .
*/
bool ToxYUVFrame : : isValid ( ) const
{
return width > 0 & & height > 0 ;
}
/**
* @ brief Checks if the given ToxYUVFrame is valid or not , delegates to isValid ( ) .
*/
ToxYUVFrame : : operator bool ( ) const
{
return { width , height } ;
return isValid ( ) ;
}
sudden6
9 years ago