invalid read in swscale when converting from rgb24 to yuv420p

[Clément Bœsch]

#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>

#define W 320
#define H 240

int main()
{
    AVPicture dst = {{0}}, src = {{0}};
    struct SwsContext *sws = sws_getCachedContext(NULL,
                                                  W, H, PIX_FMT_RGB24,
                                                  W, H, PIX_FMT_YUV420P,
                                                  0, NULL, NULL, NULL);
    if (!sws)
        return 1;
    avpicture_alloc(&src, PIX_FMT_RGB24,   W, H);
    avpicture_alloc(&dst, PIX_FMT_YUV420P, W, H);
    memset(src.data[0], 0, H * src.linesize[0]);
    if (sws_scale(sws,
                  (const uint8_t**)src.data, src.linesize, 0, H,
                  dst.data, dst.linesize) < 0)
        return 1;
    return 0;
}

☭ valgrind ./scale
==26516== Memcheck, a memory error detector
==26516== Copyright (C) 2002-2013, and GNU GPL'd, by Julian Seward et al.
==26516== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info
==26516== Command: ./scale
==26516== 
==26516== Invalid read of size 16
==26516==    at 0x442D47: ??? (input.asm:334)
==26516==    by 0x40B5A8: swscale (swscale.c:254)
==26516==    by 0x40D3E4: sws_scale (swscale.c:1101)
==26516==    by 0x40A32A: main (scale.c:19)
==26516==  Address 0x8f3dcd4 is 230,388 bytes inside a block of size 230,401 alloc'd
==26516==    at 0x4C2ACC0: memalign (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==26516==    by 0x4C2ADD7: posix_memalign (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==26516==    by 0x452BF9: av_malloc (mem.c:94)
==26516==    by 0x451586: av_image_alloc (imgutils.c:210)
==26516==    by 0x40A3D5: avpicture_alloc (avpicture.c:57)
==26516==    by 0x40A2AA: main (scale.c:16)
==26516== 
==26516== 
==26516== HEAP SUMMARY:
==26516==     in use at exit: 409,728 bytes in 34 blocks
==26516==   total heap usage: 46 allocs, 12 frees, 463,488 bytes allocated
==26516== 
==26516== LEAK SUMMARY:
==26516==    definitely lost: 375,906 bytes in 3 blocks
==26516==    indirectly lost: 33,822 bytes in 31 blocks
==26516==      possibly lost: 0 bytes in 0 blocks
==26516==    still reachable: 0 bytes in 0 blocks
==26516==         suppressed: 0 bytes in 0 blocks
==26516== Rerun with --leak-check=full to see details of leaked memory
==26516== 
==26516== For counts of detected and suppressed errors, rerun with: -v
==26516== ERROR SUMMARY: 1 errors from 1 contexts (suppressed: 3 from 3)

This might be because avpicture_alloc() calls av_image_alloc() with an align of 1.

[Carl Eugen Hoyos]

Also reproducible with the following command lines, see ticket #3582:

Invalid read of size 4:

$ valgrind ./ffmpeg_g -cpuflags mmx+sse+sse2 -f lavfi -i testsrc=s=634x320 -frames:v 1 -pix_fmt yuv420p -f null -

Invalid read of size 8:

$ valgrind ./ffmpeg_g -cpuflags mmx+sse+sse2+ssse3 -f lavfi -i testsrc=s=634x320 -frames:v 1 -vcodec mpeg4 -f null -

[Clément Bœsch]

The valgrind commands don't triggers the issue anymore. OTOH the original code still does. The following hack prevents the valgrind issue:

--- a/libavcodec/avpicture.c
+++ b/libavcodec/avpicture.c
@@ -55,7 +55,7 @@ int avpicture_alloc(AVPicture *picture,
                     enum AVPixelFormat pix_fmt, int width, int height)
 {
     int ret = av_image_alloc(picture->data, picture->linesize,
-                             width, height, pix_fmt, 1);
+                             width, height, pix_fmt, 32);
     if (ret < 0) {
         memset(picture, 0, sizeof(AVPicture));
         return ret;