mss2: fix decoding of mss2

[ami_stuff]

"fail.wmv" file decodes fine with wmp, but not with ffmpeg.

"patch.wmv" file decodes fine sine b077eb07805dc5d139b2f118ddb122cc9df8b87a and I belive that
this code must be changed to fix decoding of "fail.wmv"

commit b077eb07805dc5d139b2f118ddb122cc9df8b87a
Author: Alberto Delmás <adelmas@gmail.com>
Date:   Sun Nov 11 09:23:06 2012 +0100

    mss2: fix handling of unmasked implicit WMV9 rectangles
    
    Signed-off-by: Kostya Shishkov <kostya.shishkov@gmail.com>

diff --git a/libavcodec/mss2.c b/libavcodec/mss2.c
index 597ebb6..ceeff5a 100644
--- a/libavcodec/mss2.c
+++ b/libavcodec/mss2.c
@@ -474,7 +474,7 @@ static int mss2_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     int keyframe, has_wmv9, has_mv, is_rle, is_555, ret;
 
     Rectangle wmv9rects[MAX_WMV9_RECTANGLES], *r;
-    int used_rects = 0, i, implicit_rect, av_uninit(wmv9_mask);
+    int used_rects = 0, i, implicit_rect = 0, av_uninit(wmv9_mask);
 
     av_assert0(FF_INPUT_BUFFER_PADDING_SIZE >=
                ARITH2_PADDING + (MIN_CACHE_BITS + 7) / 8);
@@ -650,7 +650,14 @@ static int mss2_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
             return AVERROR_INVALIDDATA;
 
         buf_size -= bytestream2_tell(&gB);
-    } else if (is_rle) {
+    } else {
+        if (keyframe) {
+            c->corrupted = 0;
+            ff_mss12_slicecontext_reset(&ctx->sc[0]);
+            if (c->slice_split)
+                ff_mss12_slicecontext_reset(&ctx->sc[1]);
+        }
+    if (is_rle) {
         init_get_bits(&gb, buf, buf_size * 8);
         if (ret = decode_rle(&gb, c->pal_pic, c->pal_stride,
                              c->rgb_pic, c->rgb_stride, c->pal, keyframe,
@@ -669,14 +676,8 @@ static int mss2_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         align_get_bits(&gb);
         buf      += get_bits_count(&gb) >> 3;
         buf_size -= get_bits_count(&gb) >> 3;
-    } else {
-        if (keyframe) {
-            c->corrupted = 0;
-            ff_mss12_slicecontext_reset(&ctx->sc[0]);
-            if (c->slice_split)
-                ff_mss12_slicecontext_reset(&ctx->sc[1]);
-        }
-        else if (c->corrupted)
+    } else if (!implicit_rect || wmv9_mask != -1) {
+        if (c->corrupted)
             return AVERROR_INVALIDDATA;
         bytestream2_init(&gB, buf, buf_size + ARITH2_PADDING);
         arith2_init(&acoder, &gB);
@@ -702,6 +703,8 @@ static int mss2_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
             buf      += arith2_get_consumed_bytes(&acoder);
             buf_size -= arith2_get_consumed_bytes(&acoder);
         }
+    } else
+        memset(c->pal_pic, 0, c->pal_stride * avctx->height);
     }
 
     if (has_wmv9) {

C:\>ffmpeg -i fail.wmv -f null -
ffmpeg version N-66438-g4f4f08e Copyright (c) 2000-2014 the FFmpeg developers
  built on Sep 24 2014 22:26:49 with gcc 4.9.1 (GCC)
  configuration: --enable-gpl --enable-version3 --disable-w32threads --enable-av
isynth --enable-bzlib --enable-fontconfig --enable-frei0r --enable-gnutls --enab
le-iconv --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --
enable-libfreetype --enable-libgme --enable-libgsm --enable-libilbc --enable-lib
modplug --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrw
b --enable-libopenjpeg --enable-libopus --enable-librtmp --enable-libschroedinge
r --enable-libsoxr --enable-libspeex --enable-libtheora --enable-libtwolame --en
able-libvidstab --enable-libvo-aacenc --enable-libvo-amrwbenc --enable-libvorbis
 --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx264 --enable-
libx265 --enable-libxavs --enable-libxvid --enable-zlib
  libavutil      54.  7.101 / 54.  7.101
  libavcodec     56.  1.101 / 56.  1.101
  libavformat    56.  5.100 / 56.  5.100
  libavdevice    56.  1.100 / 56.  1.100
  libavfilter     5.  1.102 /  5.  1.102
  libswscale      3.  1.100 /  3.  1.100
  libswresample   1.  1.100 /  1.  1.100
  libpostproc    53.  1.100 / 53.  1.100
Guessed Channel Layout for  Input Stream #0.0 : stereo
Input #0, asf, from 'fail.wmv':
  Metadata:
    SfOriginalFPS   : 300000
    WMFSDKVersion   : 9.00.00.4509
    WMFSDKNeeded    : 0.0.0.0000
    Buffer Average  : 2566
    VBR Peak        : 216276
    IsVBR           : 1
    DeviceConformanceTemplate: @
  Duration: 00:00:05.03, start: 0.000000, bitrate: 565 kb/s
    Stream #0:0(pol): Audio: wmav2 (a[1][0][0] / 0x0161), 44100 Hz, 2 channels,
fltp, 128 kb/s
    Stream #0:1(pol): Video: mss2 (MSS2 / 0x3253534D), rgb24, 720x480, 414 kb/s,
 SAR 10:11 DAR 15:11, 1k tbr, 1k tbn, 1k tbc
Output #0, null, to 'pipe:':
  Metadata:
    SfOriginalFPS   : 300000
    WMFSDKVersion   : 9.00.00.4509
    WMFSDKNeeded    : 0.0.0.0000
    Buffer Average  : 2566
    VBR Peak        : 216276
    IsVBR           : 1
    DeviceConformanceTemplate: @
    encoder         : Lavf56.5.100
    Stream #0:0(pol): Video: rawvideo (RGB[24] / 0x18424752), rgb24, 720x480 [SA
R 10:11 DAR 15:11], q=2-31, 200 kb/s, 1k fps, 1k tbn, 1k tbc
    Metadata:
      encoder         : Lavc56.1.101 rawvideo
    Stream #0:1(pol): Audio: pcm_s16le, 44100 Hz, stereo, s16, 1411 kb/s
    Metadata:
      encoder         : Lavc56.1.101 pcm_s16le
Stream mapping:
  Stream #0:1 -> #0:0 (mss2 (native) -> rawvideo (native))
  Stream #0:0 -> #0:1 (wmav2 (native) -> pcm_s16le (native))
Press [q] to stop, [?] for help
[mss2 @ 031b19a0] buffer not fully consumed
[null @ 02e4ee60] Encoder did not produce proper pts, making some up.
[mss2 @ 031b19a0] buffer not fully consumed
frame=    2 fps=0.0 q=0.0 Lsize=N/A time=00:00:05.01 bitrate=N/A
video:0kB audio:872kB subtitle:0kB other streams:0kB global headers:0kB muxing o
verhead: unknown

[ami_stuff]

see also #3255

[ami_stuff]

when input file is jpeg for windows media encoder then ffmpeg can decode output wmv file, when input file for wme is bmp then output file is not decodable by ffmpeg :/

encoding settings were the same for both files

[Carl Eugen Hoyos]

Fixed by Peter Ross in 31162eb9490e51a526ff0f78a57303a4d6433aba