sws_scale crash with PIX_FMT_YUVJ420P -> PIX_FMT_RGB24

[rich99]

Last ffmpeg built from git, 10/20/2011 10:22:50AM.

Win7 x64, VS2010, built using migGW and msys, gcc 4.5.2.

Build options:
./configure --enable-shared --disable-static --enable-memalign-hack --target-os=mingw32 --arch=i686 --cpu=i686 --enable-avisynth --enable-zlib --enable-bzlib --enable-pthreads --enable-runtime-cpudetect --disable-encoders --enable-swscale --extra-cflags=-USTRICT_ANSI

Everything works fine with playback except some crashes when using sws_scale and certain video files. Here is an example that always crashes with a sample code provided. I have stripped all checks to simplify the code to the maximum. The sample video is also attached. If I don't use sws_scale, everything works fine.

main.cpp

#define __STDC_CONSTANT_MACROS
#pragma warning(disable:4244)

extern "C" 
{
#include <stdint.h>
#include <inttypes.h>

#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavcodec/avcodec.h"
#include <libavutil/opt.h>
}

int main()
{
	av_register_all();

	AVFormatContext* pFormatCtx;
	av_open_input_file(&pFormatCtx, "ffdmjpeg.avi", NULL, 0, NULL);
	av_find_stream_info(pFormatCtx);
	dump_format(pFormatCtx, 0, "test", 0);

	AVCodecContext* pCodecCtx;

	// i know first stream is at index 0, simplified here
	unsigned int videoStream = 0;
	pCodecCtx = pFormatCtx->streams[videoStream]->codec;

	AVCodec* pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
	avcodec_open(pCodecCtx, pCodec);

	AVFrame* pFrame = avcodec_alloc_frame();
	AVFrame* pFrameRGB = avcodec_alloc_frame();

	PixelFormat destFormat = PIX_FMT_RGB24;

	// Determine required buffer size and allocate buffer
	int numBytes = avpicture_get_size(destFormat, pCodecCtx->width, pCodecCtx->height);
	uint8_t* buffer = (uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
	avpicture_fill((AVPicture *)pFrameRGB, buffer, destFormat, pCodecCtx->width, pCodecCtx->height);

	int frameFinished;
	AVPacket packet;

	SwsContext* swsContext = sws_alloc_context();
	//AVOption options;
	av_opt_set_defaults(swsContext);
	av_set_int(swsContext, "sws_flags", SWS_POINT);
	av_set_int(swsContext, "srcw", pCodecCtx->width);
	av_set_int(swsContext, "srch", pCodecCtx->height);
	av_set_int(swsContext, "dstw", pCodecCtx->width);
	av_set_int(swsContext, "dsth", pCodecCtx->height);
	av_set_int(swsContext, "src_format", pCodecCtx->pix_fmt);
	av_set_int(swsContext, "dst_format", destFormat);
	av_set_int(swsContext, "src_range", 1);
	av_set_int(swsContext, "dst_range", 1);

	sws_init_context(swsContext, NULL, NULL); //success

	while(av_read_frame(pFormatCtx, &packet) >= 0) 
	{
		// Is this a packet from the video stream?
		if(packet.stream_index == videoStream) 
		{
			// Decode video frame
			avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);

			// Did we get a video frame?
			if(frameFinished) 
			{
					AVPicture &pic = *(AVPicture*)pFrame;
					AVPicture &picDst = *(AVPicture*)pFrameRGB;
					// [CRASH HERE]
					sws_scale(swsContext, &pic.data[0], &pic.linesize[0], 0, pCodecCtx->height, &picDst.data[0], &picDst.linesize[0]);
					// 
					printf(".");
			}
		}

		// Free the packet that was allocated by av_read_frame
		av_free_packet(&packet);
	}

	av_free(buffer);
	av_free(pFrameRGB);
	av_free(pFrame);
	sws_freeContext(swsContext);
	avcodec_close(pCodecCtx);
	av_close_input_file(pFormatCtx);

	return 0;
}

[rich99]

File used in code that causes crash.

[rich99]

Crash also happens with older versions of sws_scale and ffmpeg. Source code has been attached and was compiled and run with VS2010 on intel i7 2600K.

[Carl Eugen Hoyos]

Please provide backtrace etc. as explained on ​http://ffmpeg.org/bugreports.html and please confirm that the crash also happens if you use gcc.

[rich99]

The bug still happens with gdb. Compilation command used was the following:
gcc -g -ID:\devel\dependencies\ffmpeg_git\include sws_scale_test.cpp -o sws_d.exe -lavcodec -lavdevice -lavutil -lavformat -lswscale -LD:\devel\dependencies\ffmpeg_git\liblin

The backtrace was included in the attachment ffdmjpeg_crash_swscale.txt.

[rich99]

compiled this file and gave ffdmjpeg.avi as argument

[rich99]

gdb debug crash info

[rich99]

Compiled again with latest git version with following command and still crash. Files have been updated for git 10/24/2011 5:45:44PM.

gcc sws_scale_test.cpp -o ffdmjpeg -ID:/devel/dependencies/ffmpeg/ -LD:/devel/dependencies/ffmpeg/liblin -lavcodec -lavdevice -lavutil -lavformat -lswscale

[rich99]

Just to make sure, this is the final code, also attached as cpp.

#define __STDC_CONSTANT_MACROS

extern "C" 
{
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libavcodec/avcodec.h>
#include <libavutil/opt.h>
}

int main(int argc, char *argv[])
{
	av_register_all();

	AVFormatContext* pFormatCtx;
	if (argc == 2)
	{
		av_open_input_file(&pFormatCtx, argv[1], NULL, 0, NULL);
	}
	else
	{
		printf("need filename\n");
		
		return -1;
	}
	
	av_find_stream_info(pFormatCtx);
	dump_format(pFormatCtx, 0, "test", 0);

	AVCodecContext* pCodecCtx;

	// i know first stream is at index 0, simplified here
	unsigned int videoStream = 0;
	pCodecCtx = pFormatCtx->streams[videoStream]->codec;

	AVCodec* pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
	avcodec_open(pCodecCtx, pCodec);

	AVFrame* pFrame = avcodec_alloc_frame();
	AVFrame* pFrameRGB = avcodec_alloc_frame();

	PixelFormat destFormat = PIX_FMT_YUV420P;

	// Determine required buffer size and allocate buffer
	int numBytes = avpicture_get_size(destFormat, pCodecCtx->width, pCodecCtx->height);
	uint8_t* buffer = (uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
	avpicture_fill((AVPicture *)pFrameRGB, buffer, destFormat, pCodecCtx->width, pCodecCtx->height);

	int frameFinished;
	AVPacket packet;

	SwsContext* swsContext = sws_alloc_context();
	//AVOption options;
	av_opt_set_defaults(swsContext);
	av_set_int(swsContext, "sws_flags", SWS_POINT);
	av_set_int(swsContext, "srcw", pCodecCtx->width);
	av_set_int(swsContext, "srch", pCodecCtx->height);
	av_set_int(swsContext, "dstw", pCodecCtx->width);
	av_set_int(swsContext, "dsth", pCodecCtx->height);
	av_set_int(swsContext, "src_format", pCodecCtx->pix_fmt);
	av_set_int(swsContext, "dst_format", destFormat);
	av_set_int(swsContext, "src_range", 1);
	av_set_int(swsContext, "dst_range", 1);

	sws_init_context(swsContext, NULL, NULL); //success

	while(av_read_frame(pFormatCtx, &packet) >= 0) 
	{
		// Is this a packet from the video stream?
		if(packet.stream_index == videoStream) 
		{
			// Decode video frame
			avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);

			// Did we get a video frame?
			if(frameFinished) 
			{
					AVPicture &pic = *(AVPicture*)pFrame;
					AVPicture &picDst = *(AVPicture*)pFrameRGB;
					// [CRASH HERE]
					sws_scale(swsContext, &pic.data[0], &pic.linesize[0], 0, pCodecCtx->height, &picDst.data[0], &picDst.linesize[0]);
					// 
					printf(".");
			}
		}

		// Free the packet that was allocated by av_read_frame
		av_free_packet(&packet);
	}

	av_free(buffer);
	av_free(pFrameRGB);
	av_free(pFrame);
	sws_freeContext(swsContext);
	avcodec_close(pCodecCtx);
	av_close_input_file(pFormatCtx);

	return 0;
}