Ticket #4437: avf.patch

File avf.patch, 11.6 KB (added by Thilo Borgmann, 11 years ago)

First attempt to solve the issue send to ffmpeg-devel

  • libavdevice/avfoundation.m

     ---
     libavdevice/avfoundation.m | 154 ++++++++++++++++++++++++++-------------------
     1 file changed, 91 insertions(+), 63 deletions(-)
    
    diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
    index 763e675..1ed9cef 100644
    a b  
    3737#include "libavutil/time.h"
    3838#include "avdevice.h"
    3939
     40#define QUEUE_SIZE 4
     41
    4042static const int avf_time_base = 1000000;
    4143
    4244static const AVRational avf_time_base_q = {
    … … static const struct AVFPixelFormatSpec avf_pixel_formats[] = {  
    7880    { AV_PIX_FMT_NONE, 0 }
    7981};
    8082
     83enum {
     84    QUEUE_IS_EMPTY,
     85    QUEUE_HAS_BUFFERS,
     86};
     87
    8188typedef struct
    8289{
    8390    AVClass*        class;
    … … typedef struct  
    8693    int             audio_frames_captured;
    8794    int64_t         first_pts;
    8895    int64_t         first_audio_pts;
    89     pthread_mutex_t frame_lock;
    90     pthread_cond_t  frame_wait_cond;
    9196    id              avf_delegate;
    9297    id              avf_audio_delegate;
    9398
    … … typedef struct  
    124129    AVCaptureSession         *capture_session;
    125130    AVCaptureVideoDataOutput *video_output;
    126131    AVCaptureAudioDataOutput *audio_output;
    127     CMSampleBufferRef         current_frame;
    128     CMSampleBufferRef         current_audio_frame;
    129 } AVFContext;
    130132
    131 static void lock_frames(AVFContext* ctx)
    132 {
    133     pthread_mutex_lock(&ctx->frame_lock);
    134 }
     133    NSConditionLock *lock;
     134    NSMutableArray *video_queue;
     135    NSMutableArray *audio_queue;
    135136
    136 static void unlock_frames(AVFContext* ctx)
    137 {
    138     pthread_mutex_unlock(&ctx->frame_lock);
    139 }
     137} AVFContext;
    140138
    141139/** FrameReciever class - delegate for AVCaptureSession
    142140 */
    … … static void unlock_frames(AVFContext* ctx)  
    167165  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
    168166         fromConnection:(AVCaptureConnection *)connection
    169167{
    170     lock_frames(_context);
     168    NSMutableArray *queue = _context->video_queue;
     169    NSConditionLock *lock = _context->lock;
    171170
    172     if (_context->current_frame != nil) {
    173         CFRelease(_context->current_frame);
    174     }
     171    [lock lock];
    175172
    176     _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
     173    if ([queue count] == QUEUE_SIZE) {
     174        av_log(_context, AV_LOG_WARNING, "video queue is full, the oldest frame has been dropped\n");
     175        [queue removeLastObject];
     176    }
    177177
    178     pthread_cond_signal(&_context->frame_wait_cond);
     178    [queue insertObject:(id)videoFrame atIndex:0];
    179179
    180     unlock_frames(_context);
     180    [lock unlockWithCondition:QUEUE_HAS_BUFFERS];
    181181
    182182    ++_context->frames_captured;
    183183}
    … … static void unlock_frames(AVFContext* ctx)  
    213213  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
    214214         fromConnection:(AVCaptureConnection *)connection
    215215{
    216     lock_frames(_context);
     216    NSMutableArray *queue = _context->audio_queue;
     217    NSConditionLock *lock = _context->lock;
    217218
    218     if (_context->current_audio_frame != nil) {
    219         CFRelease(_context->current_audio_frame);
    220     }
     219    [lock lock];
    221220
    222     _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
     221    if ([queue count] == QUEUE_SIZE) {
     222        av_log(_context, AV_LOG_WARNING, "audio queue is full, the oldest frame has been dropped\n");
     223        [queue removeLastObject];
     224    }
    223225
    224     pthread_cond_signal(&_context->frame_wait_cond);
     226    [queue insertObject:(id)audioFrame atIndex:0];
    225227
    226     unlock_frames(_context);
     228    [lock unlockWithCondition:QUEUE_HAS_BUFFERS];
    227229
    228230    ++_context->audio_frames_captured;
    229231}
    … … static void destroy_context(AVFContext* ctx)  
    248250
    249251    av_freep(&ctx->audio_buffer);
    250252
    251     pthread_mutex_destroy(&ctx->frame_lock);
    252     pthread_cond_destroy(&ctx->frame_wait_cond);
     253    [ctx->audio_queue release];
     254    [ctx->video_queue release];
     255    [ctx->lock release];
    253256
    254     if (ctx->current_frame) {
    255         CFRelease(ctx->current_frame);
    256     }
     257    ctx->audio_queue = NULL;
     258    ctx->video_queue = NULL;
     259    ctx->lock = NULL;
    257260}
    258261
    259262static void parse_device_name(AVFormatContext *s)
    … … static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)  
    538541static int get_video_config(AVFormatContext *s)
    539542{
    540543    AVFContext *ctx = (AVFContext*)s->priv_data;
     544    CMSampleBufferRef sample_buffer;
    541545    CVImageBufferRef image_buffer;
    542546    CGSize image_buffer_size;
    543547    AVStream* stream = avformat_new_stream(s, NULL);
    … … static int get_video_config(AVFormatContext *s)  
    551555        CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
    552556    }
    553557
    554     lock_frames(ctx);
     558    [ctx->lock lock];
    555559
    556560    ctx->video_stream_index = stream->index;
    557561
    558562    avpriv_set_pts_info(stream, 64, 1, avf_time_base);
    559563
    560     image_buffer      = CMSampleBufferGetImageBuffer(ctx->current_frame);
     564    sample_buffer     = (CMSampleBufferRef)[ctx->video_queue lastObject];
     565    image_buffer      = CMSampleBufferGetImageBuffer(sample_buffer);
    561566    image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
    562567
    563568    stream->codec->codec_id   = AV_CODEC_ID_RAWVIDEO;
    … … static int get_video_config(AVFormatContext *s)  
    566571    stream->codec->height     = (int)image_buffer_size.height;
    567572    stream->codec->pix_fmt    = ctx->pixel_format;
    568573
    569     CFRelease(ctx->current_frame);
    570     ctx->current_frame = nil;
     574    [ctx->lock unlockWithCondition:QUEUE_HAS_BUFFERS];
    571575
    572     unlock_frames(ctx);
     576    [ctx->video_queue removeLastObject];
    573577
    574578    return 0;
    575579}
    … … static int get_video_config(AVFormatContext *s)  
    577581static int get_audio_config(AVFormatContext *s)
    578582{
    579583    AVFContext *ctx = (AVFContext*)s->priv_data;
     584    CMSampleBufferRef sample_buffer;
    580585    CMFormatDescriptionRef format_desc;
    581586    AVStream* stream = avformat_new_stream(s, NULL);
    582587
    … … static int get_audio_config(AVFormatContext *s)  
    589594        CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
    590595    }
    591596
    592     lock_frames(ctx);
     597    [ctx->lock lock];
    593598
    594599    ctx->audio_stream_index = stream->index;
    595600
    596601    avpriv_set_pts_info(stream, 64, 1, avf_time_base);
    597602
    598     format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
     603    sample_buffer = (CMSampleBufferRef)[ctx->audio_queue lastObject];
     604    format_desc = CMSampleBufferGetFormatDescription(sample_buffer);
    599605    const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
    600606
    601607    if (!basic_desc) {
    … … static int get_audio_config(AVFormatContext *s)  
    642648    }
    643649
    644650    if (ctx->audio_non_interleaved) {
    645         CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
     651        CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(sample_buffer);
    646652        ctx->audio_buffer_size        = CMBlockBufferGetDataLength(block_buffer);
    647653        ctx->audio_buffer             = av_malloc(ctx->audio_buffer_size);
    648654        if (!ctx->audio_buffer) {
    … … static int get_audio_config(AVFormatContext *s)  
    651657        }
    652658    }
    653659
    654     CFRelease(ctx->current_audio_frame);
    655     ctx->current_audio_frame = nil;
    656 
    657     unlock_frames(ctx);
     660    [ctx->lock unlockWithCondition:QUEUE_HAS_BUFFERS];
    658661
    659662    return 0;
    660663}
    … … static int avf_read_header(AVFormatContext *s)  
    674677    ctx->first_pts          = av_gettime();
    675678    ctx->first_audio_pts    = av_gettime();
    676679
    677     pthread_mutex_init(&ctx->frame_lock, NULL);
    678     pthread_cond_init(&ctx->frame_wait_cond, NULL);
     680    ctx->lock = [[NSConditionLock alloc] initWithCondition:QUEUE_IS_EMPTY];
     681    ctx->video_queue = [[NSMutableArray alloc] initWithCapacity:QUEUE_SIZE];
     682    ctx->audio_queue = [[NSMutableArray alloc] initWithCapacity:QUEUE_SIZE];
    679683
    680684#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
    681685    CGGetActiveDisplayList(0, NULL, &num_screens);
    … … static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)  
    897901    AVFContext* ctx = (AVFContext*)s->priv_data;
    898902
    899903    do {
    900         CVImageBufferRef image_buffer;
    901         lock_frames(ctx);
     904        int got_buffer = 0;
     905        CMSampleBufferRef asample_buffer;
     906        CMSampleBufferRef vsample_buffer;
     907
     908        [ctx->lock lockWhenCondition:QUEUE_HAS_BUFFERS];
     909        vsample_buffer = (CMSampleBufferRef)[ctx->video_queue lastObject];
     910        if (vsample_buffer) {
     911            vsample_buffer = (CMSampleBufferRef)CFRetain(vsample_buffer);
     912            [ctx->video_queue removeLastObject];
     913
     914            got_buffer |= 1;
     915        }
     916
     917        asample_buffer = (CMSampleBufferRef)[ctx->audio_queue lastObject];
     918        if (asample_buffer) {
     919            asample_buffer = (CMSampleBufferRef)CFRetain(asample_buffer);
     920            [ctx->audio_queue removeLastObject];
     921
     922            got_buffer |= 1;
     923        }
    902924
    903         image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
     925        if (!got_buffer || ([ctx->video_queue count] == 0 && [ctx->audio_queue count] == 0)) {
     926            [ctx->lock unlockWithCondition:QUEUE_IS_EMPTY];
     927        } else {
     928            [ctx->lock unlock];
     929        }
    904930
    905         if (ctx->current_frame != nil) {
     931        if (vsample_buffer != nil) {
    906932            void *data;
     933            CVImageBufferRef image_buffer;
     934
     935            image_buffer = CMSampleBufferGetImageBuffer(vsample_buffer);
    907936            if (av_new_packet(pkt, (int)CVPixelBufferGetDataSize(image_buffer)) < 0) {
     937                CVPixelBufferUnlockBaseAddress(image_buffer, 0);
     938                CFRelease(vsample_buffer);
    908939                return AVERROR(EIO);
    909940            }
    910941
    911942            CMItemCount count;
    912943            CMSampleTimingInfo timing_info;
    913944
    914             if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
     945            if (CMSampleBufferGetOutputSampleTimingInfoArray(vsample_buffer, 1, &timing_info, &count) == noErr) {
    915946                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
    916947                pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
    917948            }
    … … static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)  
    925956            memcpy(pkt->data, data, pkt->size);
    926957
    927958            CVPixelBufferUnlockBaseAddress(image_buffer, 0);
    928             CFRelease(ctx->current_frame);
    929             ctx->current_frame = nil;
    930         } else if (ctx->current_audio_frame != nil) {
    931             CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
     959            CFRelease(vsample_buffer);
     960            vsample_buffer = NULL;
     961        }
     962
     963        if (asample_buffer != nil) {
     964            CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(asample_buffer);
    932965            int block_buffer_size         = CMBlockBufferGetDataLength(block_buffer);
    933966
    934967            if (!block_buffer || !block_buffer_size) {
    … … static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)  
    946979            CMItemCount count;
    947980            CMSampleTimingInfo timing_info;
    948981
    949             if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
     982            if (CMSampleBufferGetOutputSampleTimingInfoArray(asample_buffer, 1, &timing_info, &count) == noErr) {
    950983                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
    951984                pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
    952985            }
    … … static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)  
    9941027                }
    9951028            }
    9961029
    997             CFRelease(ctx->current_audio_frame);
    998             ctx->current_audio_frame = nil;
    999         } else {
    1000             pkt->data = NULL;
    1001             pthread_cond_wait(&ctx->frame_wait_cond, &ctx->frame_lock);
     1030            CFRelease(asample_buffer);
     1031            asample_buffer = NULL;
    10021032        }
    1003 
    1004         unlock_frames(ctx);
    10051033    } while (!pkt->data);
    10061034
    10071035    return 0;