yuv420_bgr32_ssse3 writes out of buffer boundary during sws_scale() call

[Sean]

Summary of the bug:
yuv420_bgr32_ssse3 scaling writes out of bound during sws_scale() function call.

observations:

1. Some experiments indicated that there is no out of bound writes observed when using the old mmx optimized implementation instead.
2. this issue occurs on windows, macOS, and linux builds.

sample code:
-- BEGIN --

#include <stdlib.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/log.h>
#include <libswscale/swscale.h>

void logFromSWS(char* str) {
  char buf[1024];
  memset(buf, 0, 1024);
  sprintf(buf, "logfromsws: %s\n", str);
  printf(buf);
}

int main(int argc, char** argv) {
  av_log_set_level(AV_LOG_QUIET);
  av_register_all();

  //sws_register_logging(logFromSWS);

  // new video
  AVFormatContext* format_ctx = avformat_alloc_context();
  if ( format_ctx == NULL ) {
    printf("failed to allocate context\n");
    return 1;
  }

  int rc = avformat_open_input(&format_ctx, argv[1], NULL, NULL);
  if ( rc != 0 ) {
    printf("failed to open input\n");
    return 1;
  }

  if (format_ctx->nb_streams == 0) {
    printf("no video stream found\n");
    return 1;
  }

  struct AVStream* videoStream = NULL;
  int videoStreamIndex = -1;

  for ( int i = 0; i < format_ctx->nb_streams; i++ ) {
    struct AVStream* v = format_ctx->streams[i];
    if ( v == NULL ) {
      continue;
    }
    if ( v->codec->codec_type == AVMEDIA_TYPE_VIDEO ) {
      videoStream = v;
      videoStreamIndex = i;
      printf("streams=%d, vid index=%d\n", format_ctx->nb_streams, i);
      break;
    }
  }

  if ( videoStreamIndex == -1 ) {
    printf("failed to find video stream\n");
    return 1;
  }

  int width = videoStream->codec->width;
  int height = videoStream->codec->height;

  printf("width=%d, height=%d\n", width, height);

  // extractFrameAt

  AVCodecContext* video_codec_context = videoStream->codec;
  if ( video_codec_context == NULL ) {
    printf("no video codec context found\n");
    return 1;
  }

  AVCodec* video_codec = avcodec_find_decoder(video_codec_context->codec_id);
  if ( video_codec == NULL ) {
    printf("failed to find decoder\n");
    return 1;
  }

  rc = avcodec_open2(video_codec_context, video_codec, NULL);
  if ( rc != 0 ) {
    printf("failed to open video");
    return 1;
  }

  // seek?

  // read next frame
  AVFrame* frame = av_frame_alloc();
  if ( frame == NULL ) {
    printf("failed to allocate frame\n");
    return 1;
  }

  int frame_finished;
  int iteration = 0;
  while(1) {
    printf("read frame iter=%d\n", iteration++);
    struct AVPacket packet;
    rc = av_read_frame(format_ctx, &packet);
    if ( rc != 0 ) {
      printf("failed to read frame\n");
      return 1;
    }

    if ( packet.stream_index != videoStreamIndex ) {
      av_packet_unref(&packet);
      continue;
    }

    rc = avcodec_decode_video2(video_codec_context, frame, &frame_finished, &packet);
    if ( rc < 0 ) {
      printf("failed to decode video %s\n", av_err2str(rc));
      return 1;
    }

    av_packet_unref(&packet);

    if ( frame_finished != 0 ) {
      break;
    }
  }

  // image from frame
  struct SwsContext* scaling_context = sws_getContext(width, height, (enum AVPixelFormat)frame->format, width, height, AV_PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL);

  if ( scaling_context == NULL ) {
    printf("failed to get scaling context\n");
    return 1;
  }

  //uint8_t** src_slice = &(frame->data[0]);
  //int* src_stride = &(frame->linesize[0]);

  AVFrame *frame_rgb;
  //frame_rgb = avcodec_alloc_frame();
  frame_rgb = av_frame_alloc();
  uint8_t* buffer_rgb;
  int nbytes = avpicture_get_size(AV_PIX_FMT_RGBA, width, height);
  size_t baseAllocSize = nbytes * sizeof(uint8_t);
  size_t allocSize = baseAllocSize + 32;

  if ( baseAllocSize != (width * height * 4 )) {
    printf("invalid allocation size\n");
    return 1;
  }

  buffer_rgb = (uint8_t*)av_malloc(allocSize);
  memset(buffer_rgb, 0, nbytes);
  avpicture_fill((AVPicture *)frame_rgb, buffer_rgb, AV_PIX_FMT_RGBA, width, height);

  printf("buffer size=%d, height=%d, linesize=%d\n", nbytes, height, frame_rgb->linesize[0]);

  if ( frame_rgb->linesize[0] != baseAllocSize / height ) {
    printf("unexpected stride size\n");
    return 1;
  }

  for ( int i = baseAllocSize; i < allocSize; i++ ) {
    if ( buffer_rgb[i] != 0 ) {
      printf("buffer index %d is not zero\n", i);
      return 1;
    }
  }

  rc = sws_scale(scaling_context, frame->data, frame->linesize, 0, height, frame_rgb->data, frame_rgb->linesize);
  if ( rc <= 0 ) {
    printf("failed to scale %d\n", rc);
    return 1;
  }

  for ( int i = baseAllocSize; i < allocSize; i++ ) {
    if ( buffer_rgb[i] != 0 ) {
      printf("buffer index %d is not zero\n", i);
    }
  }

  printf("succeeded %d\n", rc);
  return 0;
}

-- END --

Output:
-- BEGIN --
streams=2, vid index=0
width=1080, height=1280
read frame iter=0
buffer size=5529600, height=1280, linesize=4320
buffer index 5529603 is not zero
buffer index 5529607 is not zero
buffer index 5529611 is not zero
buffer index 5529615 is not zero
buffer index 5529619 is not zero
buffer index 5529623 is not zero
buffer index 5529627 is not zero
buffer index 5529631 is not zero
succeeded 1280
-- END --

The output indicated that yuv420_bgr32_ssse3 ended up writing another 32 bytes past the buffer boundary.

How to reproduce:

The following steps have been reproduced on 4.4.1 and 4.4.2.

1. change line 140 of the sample code to size_t allocSize = baseAllocSize);
2. build and execute in valgrind with argv[1] being the input file
3. observe output from within valgrind:

valgrind ./video_extract_linux /userhome/Downloads/ASF.asf
==141== Memcheck, a memory error detector
==141== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.
==141== Using Valgrind-3.14.0 and LibVEX; rerun with -h for copyright info
==141== Command: ./video_extract_linux /userhome/Downloads/ASF.asf
==141==
streams=2, vid index=0
width=1080, height=1280
read frame iter=0
buffer size=5529600, height=1280, linesize=4320
==141== Invalid write of size 8
==141== at 0x608BB33: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x608A38A: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x605E45F: sws_scale (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x10986A: main (in /userhome/projects/video_extract_linux)
==141== Address 0x71bf080 is 0 bytes after a block of size 5,529,600 alloc'd
==141== at 0x4837EC3: memalign (vg_replace_malloc.c:898)
==141== by 0x4837FF0: posix_memalign (vg_replace_malloc.c:1062)
==141== by 0x5B2CCFE: av_malloc (in /deps/amd64/lib/libavutil.so.56.70.100)
==141== by 0x109754: main (in /userhome/projects/video_extract_linux)
==141==
==141== Invalid write of size 8
==141== at 0x608BB38: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x608A38A: ??? (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x605E45F: sws_scale (in /deps/amd64/lib/libswscale.so.5.9.100)
==141== by 0x10986A: main (in /userhome/projects/video_extract_linux)
==141== Address 0x71bf090 is 16 bytes after a block of size 5,529,600 alloc'd
==141== at 0x4837EC3: memalign (vg_replace_malloc.c:898)
==141== by 0x4837FF0: posix_memalign (vg_replace_malloc.c:1062)
==141== by 0x5B2CCFE: av_malloc (in /deps/amd64/lib/libavutil.so.56.70.100)
==141== by 0x109754: main (in /userhome/projects/video_extract_linux)
==141==
succeeded 1280
==141==
==141== HEAP SUMMARY:
==141== in use at exit: 10,183,639 bytes in 193 blocks
==141== total heap usage: 233 allocs, 40 frees, 10,462,744 bytes allocated
==141==
==141== LEAK SUMMARY:
==141== definitely lost: 55,584 bytes in 4 blocks
==141== indirectly lost: 10,128,055 bytes in 189 blocks
==141== possibly lost: 0 bytes in 0 blocks
==141== still reachable: 0 bytes in 0 blocks
==141== suppressed: 0 bytes in 0 blocks
==141== Rerun with --leak-check=full to see details of leaked memory
==141==
==141== For counts of detected and suppressed errors, rerun with: -v
==141== ERROR SUMMARY: 4 errors from 2 contexts (suppressed: 0 from 0)