Failure to write last packet to stream

[pixelmyr]

Hi there,

I'm using the FFmpeg API to take a bitmap and encode it to a video stream. The individual video chunks must be sent over a WebSocket, so I implemented a custom AVIOContext in order to the write callbacks and pass these to a parent C# application which handles the networking.

During testing, I wanted to only send a video that was only a single frame long. However, I noticed that my write callback was only fired to write the header, and was not fired for the frame I sent to be encoded. When I sent two frames, only the first of the frame was encoded along with the header, and so on. It appears that the last frame I send is never encoded.

I am using VP9 encoding on yuv420p data with a webm muxer, but this bug appears to occur with other formats as well.

My initialization code is below:

HRESULT WINAPI InitializeVideoEncoding(Encoder* encoder,
    LPCSTR codec, LPCSTR outputContainer, LPCSTR* encoderOptions, UINT encoderOptCount,
    LPCSTR* muxerOptions, UINT muxerOptCount, LPCSTR* miscOptions, UINT miscOptCount)
{
    // Fill the encoder options
    Log("Loading encoder options.");
    LoadDictionary(&encoder->encoderOptions, encoderOptions, encoderOptCount);
    if (!encoder->encoderOptions && encoderOptCount > 0)
    {
        Log("Failed to initialize encoder options.");
        return E_FAIL;
    }

    Log("Loading muxer options.");
    LoadDictionary(&encoder->muxerOptions, muxerOptions, muxerOptCount);
    if (!encoder->muxerOptions && muxerOptCount > 0)
    {
        Log("Failed to initialize encoder options.");
        return E_FAIL;
    }

    Log("Loading misc options.");
    LoadDictionary(&encoder->miscOptions, miscOptions, miscOptCount);
    if (!encoder->miscOptions && muxerOptCount > 0)
    {
        Log("Failed to initialize misc options.");
        return E_FAIL;
    }

    // Create the output context
    // This is effectively the muxer we use for the stream
    avformat_alloc_output_context2(&encoder->outputFormatContext, NULL, outputContainer, NULL);
    
    #if _DEBUG
    encoder->outputFormatContext->debug |= FF_FDEBUG_TS;
    #endif

    if (!encoder->outputFormatContext)
    {
        Log("Couldn't create output format context.");
        return E_FAIL;
    }
    encoder->outputFormat = encoder->outputFormatContext->oformat;

    // need to manually specify some parameters
    if (av_opt_set_dict(encoder->outputFormatContext->priv_data,
                        &encoder->muxerOptions) < 0)
    {
        Log("Failed to set muxer params.");
        return E_FAIL;
    }

    if (av_dict_count(encoder->muxerOptions) > 0)
    {
        Log("Dumping unused muxer params:");
        DumpDictionary(encoder->muxerOptions);
    }

    // Create the output stream using the above muxer
    encoder->outputStream = avformat_new_stream(encoder->outputFormatContext, NULL);
    if (!encoder->outputStream)
    {
        Log("Couldn't create output stream.");
        return E_FAIL;
    }
    encoder->outputStream->id = encoder->outputFormatContext->nb_streams - 1;
    
    // Find the codec
    encoder->codec = avcodec_find_encoder_by_name(codec);
    if (!encoder->codec)
    {
        Log("Couldn't find encoder.");
        return E_FAIL;
    }
    
    // Create the encoding context
    encoder->encodingContext = avcodec_alloc_context3(encoder->codec);
    if (!encoder->encodingContext)
    {
        Log("Couldn't create encoding context.");
        return E_FAIL;
    }

    // Set the basics
    encoder->encodingContext->width = encoder->width;
    encoder->encodingContext->height = encoder->height;

    // Open the codec, which pairs it with the encoding context
    int result = avcodec_open2(encoder->encodingContext, encoder->codec, &encoder->encoderOptions);
    if (result < 0)
    {
        LogFFmpegError(result, "Couldn't open codec.");
        return E_FAIL;
    }

    if (av_dict_count(encoder->encoderOptions) > 0)
    {
        Log("Dumping unused encoder params:");
        DumpDictionary(encoder->encoderOptions);
    }

    // Set some params afterwards
    encoder->outputStream->time_base = encoder->encodingContext->time_base;
    encoder->encodingContext->framerate = av_inv_q(encoder->encodingContext->time_base);
    encoder->outputStream->avg_frame_rate = encoder->encodingContext->framerate;

    if (encoder->outputFormat->flags & AVFMT_GLOBALHEADER)
        encoder->encodingContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

    // Copy necessary information to the stream
    result = avcodec_parameters_from_context(encoder->outputStream->codecpar,
                                             encoder->encodingContext);

    if (result < 0)
    {
        LogFFmpegError(result, "Couldn't copy stream parameters.");
        return E_FAIL;
    }

    av_dump_format(encoder->outputFormatContext, 0, NULL, 1);
    
    // Initialize IO

    // Grab the IO buffer size
    {
        const char* frameBufKey = "frame_buf_size";
        encoder->ioBufSize = 131072;
        AVDictionaryEntry* e = av_dict_get(encoder->miscOptions,
            frameBufKey,
            NULL, 0);
        if (e)
        {
            // Set the value and remove from the list.
            encoder->ioBufSize = strtol(e->value, NULL, 10);
            av_dict_set(&encoder->miscOptions, frameBufKey, NULL, 0);
        }
    }

    encoder->ioBuf = (LPBYTE)av_malloc(encoder->ioBufSize);
    Log("Encoder IO buffer size: %d", encoder->ioBufSize);

    AVIOContext* ioContext = avio_alloc_context(encoder->ioBuf,
                                                (int)encoder->ioBufSize,
                                                1,
                                                encoder,
                                                NULL,
                                                WriteStreamCallback,
                                                NULL);
    encoder->outputFormatContext->pb = ioContext;
    encoder->outputFormatContext->flags |= AVFMT_FLAG_CUSTOM_IO;

    if (av_dict_count(encoder->miscOptions) > 0)
    {
        Log("Dumping unused misc params:");
        DumpDictionary(encoder->miscOptions);
    }

    result = avformat_write_header(encoder->outputFormatContext, NULL);
    if (result < 0)
    {
        LogFFmpegError(result, "Couldn't write header.");
        return E_FAIL;
    }

    return S_OK;
}

And my IO function:

int WINAPI WriteStreamCallback(void* opaque, unsigned char* buf, int bufSize)
{
	Encoder* encoder = (Encoder*)opaque;
	return encoder->frameCallback(buf, bufSize);
}

You'll notice I make heavy use of AVDictionarys, and those values can be found here:

const char* encoderParams[] =
{
    "b", "2000000",
    "time_base", "1:15",
    "pixel_format", "yuv420p",
    "speed", "6",
    "tile-columns", "4",
    "frame-parallel", "1",
    "threads", "8",
    "static-thresh", "0",
    "deadline", "realtime",
    "lag-in-frames", "0",
    "error-resilient", "1"
};

const char* muxerParams[] =
{
    "dash", "1",
    "live", "1"
};

const char* miscParams[] =
{
    "frame_buf_size", "8192"
};

Is this a bug or did I miss something in the initialization?
Thanks!

[pixelmyr]

Hi all,

I made a simpler demo with none of my abstraction code. You'll find it below:

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

#include <libavutil/avassert.h>
#include <libavutil/channel_layout.h>
#include <libavutil/opt.h>
#include <libavutil/mathematics.h>
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>

#define FRAME_RATE 15

#define WIDTH 2040
#define HEIGHT 1536
#define BYTES_PER_PIXEL 3
#define BITS_PER_PIXEL 24

#define BUFFER_SIZE 131072

int IOWrite(void* opaque, unsigned char* buf, int bufSize)
{
    printf("Wrote %d bytes.\n", bufSize);
    return bufSize;
}

int main()
{
    av_register_all();

    AVFormatContext* outputFormatContext;
    avformat_alloc_output_context2(&outputFormatContext, NULL, "webm", NULL);

    AVOutputFormat* outputFormat = outputFormatContext->oformat;

    AVStream* stream = avformat_new_stream(outputFormatContext, NULL);
    stream->id = outputFormatContext->nb_streams - 1;

    AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_VP9);
    AVCodecContext* encodingContext = avcodec_alloc_context3(codec);
    encodingContext->width = WIDTH;
    encodingContext->height = HEIGHT;
    encodingContext->time_base = (AVRational){ 1, FRAME_RATE };
    encodingContext->pix_fmt = AV_PIX_FMT_YUV420P;

    avcodec_open2(encodingContext, codec, NULL);

    stream->time_base = encodingContext->time_base;
    encodingContext->framerate = av_inv_q(encodingContext->time_base);
    stream->avg_frame_rate = encodingContext->framerate;

    if (outputFormat->flags & AVFMT_GLOBALHEADER)
        encodingContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

    avcodec_parameters_from_context(stream->codecpar, encodingContext);

    unsigned char* buf = av_malloc(BUFFER_SIZE);

    AVIOContext* ioContext = avio_alloc_context(buf, BUFFER_SIZE, 1, NULL,
        NULL, IOWrite, NULL);

    outputFormatContext->pb = ioContext;
    outputFormatContext->flags |= AVFMT_FLAG_CUSTOM_IO;

    avformat_write_header(outputFormatContext, NULL);

    struct SwsContext* imageConverter = sws_getContext(WIDTH, HEIGHT, AV_PIX_FMT_BGR24, WIDTH, HEIGHT, AV_PIX_FMT_YUV420P, 0, NULL, NULL, NULL);

    AVFrame* yuvFrame = av_frame_alloc();
    // Set up the YUV destination frame
    yuvFrame->linesize[0] = WIDTH;
    yuvFrame->linesize[1] = WIDTH / 2;
    yuvFrame->linesize[2] = WIDTH / 2;

    int ySize = yuvFrame->linesize[0] * HEIGHT;
    int uSize = yuvFrame->linesize[1] * HEIGHT / 2;

    yuvFrame->format = AV_PIX_FMT_YUV420P;
    yuvFrame->width = WIDTH;
    yuvFrame->height = HEIGHT;

    // Allocate the data
    av_frame_get_buffer(yuvFrame, 32);

    AVFrame* sourceFrame = av_frame_alloc();

    unsigned long int pts = 0;

    int numFrames = 1;
    for (int i = 0; i < numFrames; ++i)
    {
        FILE* f;

        char buf[256];
        sprintf_s(buf, 256, "D:/output/Frame_%d.bmp", i);

        fopen_s(&f, buf, "rb");

        fseek(f, 0, SEEK_END);
        int bmpSize = ftell(f);
        fseek(f, 0, SEEK_SET);

        unsigned char* bmp = (unsigned char*)malloc(bmpSize);
        fread(bmp, 1, bmpSize, f);

        fclose(f);

        sourceFrame->data[0] = bmp;
        sourceFrame->data[1] = bmp + 1;
        sourceFrame->data[2] = bmp + 2;

        sourceFrame->linesize[0] = WIDTH * BYTES_PER_PIXEL;
        sourceFrame->linesize[1] = WIDTH * BYTES_PER_PIXEL;
        sourceFrame->linesize[2] = WIDTH * BYTES_PER_PIXEL;

        av_frame_make_writable(yuvFrame);

        sws_scale(imageConverter, sourceFrame->data, sourceFrame->linesize,
            0, HEIGHT, yuvFrame->data, yuvFrame->linesize);

        yuvFrame->pts = pts++;
        avcodec_send_frame(encodingContext, yuvFrame);

        free(bmp);
    }

    // Force a flush
    avcodec_send_frame(encodingContext, NULL);

    // grab every packet
    AVPacket p = { 0 };
    while (1)
    {
        av_packet_unref(&p);
        av_init_packet(&p);
        int result = avcodec_receive_packet(encodingContext, &p);
        if (result < 0)
        {
            break;
        }
        av_packet_rescale_ts(&p, encodingContext->time_base, stream->time_base);
        p.stream_index = stream->index;
        av_interleaved_write_frame(outputFormatContext, &p);
    }

    avio_flush(outputFormatContext->pb);

    printf("Press any key to continue...");
    getchar();

    return 0;
}

In the case of 1 packet, the callback only fired for the WebM header, with an output of Wrote 419 bytes..

When I played with the numFrames variable, I found that my original assumption was false. The frames definitely do get written, but the buffer is not necessarily flushed. I receive as many packets out as frames in, but they don't get written to the buffer.

I hope this helps!