crash decoding hevc

[Carl Eugen Hoyos]

(reported by ami_stuff)

(gdb) r -threads 1 -i fhevc5.ts -f null -
Starting program: ffmpeg_g -threads 1 -i fhevc5.ts -f null -
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
ffmpeg version N-57366-gfbdc98c Copyright (c) 2000-2013 the FFmpeg developers
  built on Oct 23 2013 23:51:57 with gcc 4.7 (SUSE Linux)
  configuration: --enable-gpl
  libavutil      52. 47.101 / 52. 47.101
  libavcodec     55. 38.101 / 55. 38.101
  libavformat    55. 19.104 / 55. 19.104
  libavdevice    55.  4.100 / 55.  4.100
  libavfilter     3. 89.100 /  3. 89.100
  libswscale      2.  5.101 /  2.  5.101
  libswresample   0. 17.104 /  0. 17.104
  libpostproc    52.  3.100 / 52.  3.100
[mpegts @ 0x17999e0] PES packet size mismatch
[mpegts @ 0x17999e0] Invalid timestamps stream=0, pts=258764, dts=259260, size=3386
[mpegts @ 0x17999e0] Invalid timestamps stream=0, pts=514084, dts=1559060, size=6100
[mpegts @ 0x17999e0] probed stream 1 failed
[mpegts @ 0x17999e0] Could not find codec parameters for stream 1 (Unknown: none): unknown codec
Consider increasing the value for the 'analyzeduration' and 'probesize' options
Input #0, mpegts, from 'fhevc5.ts':
  Duration: 00:00:12.60, start: 0.080000, bitrate: 1047 kb/s
  Program 1
    Stream #0:0[0x12d]: Video: hevc (HEVC / 0x43564548), yuv420p, 320x240, 23.98 tbr, 90k tbn, 90k tbc
  No Program
    Stream #0:1[0x13d]: Unknown: none
[New Thread 0x7ffff59eb700 (LWP 12694)]
[New Thread 0x7ffff51ea700 (LWP 12695)]
[New Thread 0x7ffff49e9700 (LWP 12696)]
[New Thread 0x7ffff41e8700 (LWP 12697)]
[New Thread 0x7ffff39e7700 (LWP 12698)]
[New Thread 0x7ffff31e6700 (LWP 12699)]
[New Thread 0x7ffff29e5700 (LWP 12700)]
[New Thread 0x7ffff21e4700 (LWP 12701)]
[New Thread 0x7ffff19e3700 (LWP 12702)]
Output #0, null, to 'pipe:':
  Metadata:
    encoder         : Lavf55.19.104
    Stream #0:0: Video: rawvideo (I420 / 0x30323449), yuv420p, 320x240, q=2-31, 200 kb/s, 90k tbn, 23.98 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (hevc -> rawvideo)
Press [q] to stop, [?] for help
[null @ 0x1864320] Encoder did not produce proper pts, making some up.
[hevc @ 0x179da20] Could not find ref with POC 12
[hevc @ 0x179da20] Could not find ref with POC 9
[hevc @ 0x179da20] Duplicate POC in a sequence: 13.
[hevc @ 0x179da20] Error parsing NAL unit #0.
[hevc @ 0x179da20] Invalid NAL unit 1, skipping.
[hevc @ 0x179da20] Could not find ref with POC 42
[hevc @ 0x179da20] Could not find ref with POC 51
[mpegts @ 0x17999e0] PES packet size mismatch
[mpegts @ 0x17999e0] Invalid timestamps stream=0, pts=258764, dts=259260, size=3386
[hevc @ 0x179da20] Could not find ref with POC 7
[hevc @ 0x179da20] Could not find ref with POC 59
[hevc @ 0x179da20] Could not find ref with POC 56
[hevc @ 0x179da20] Duplicate POC in a sequence: 60.
[hevc @ 0x179da20] Error parsing NAL unit #0.
[hevc @ 0x179da20] vps_max_dec_pic_buffering_minus1 out of range: 163
[hevc @ 0x179da20] Error parsing NAL unit #0.
[hevc @ 0x179da20] Ignoring POC change between slices: 0 -> 4

Program received signal SIGSEGV, Segmentation fault.
0x0000000000867790 in put_hevc_qpel_h3_8 (dst=0x7fffffff9190, dststride=0,
    _src=0x9d <Address 0x9d out of bounds>, _srcstride=0, width=32, height=16,
    mcbuffer=0x18066b0) at libavcodec/hevcdsp_template.c:910
910     PUT_HEVC_QPEL_H(3)
(gdb) bt
#0  0x0000000000867790 in put_hevc_qpel_h3_8 (dst=0x7fffffff9190, dststride=0,
    _src=0x9d <Address 0x9d out of bounds>, _srcstride=0, width=32, height=16,
    mcbuffer=0x18066b0) at libavcodec/hevcdsp_template.c:910
#1  0x000000000084749b in hls_prediction_unit (s=s@entry=0x178a9e0, x0=x0@entry=160,
    y0=y0@entry=96, nPbW=nPbW@entry=32, nPbH=nPbH@entry=16, log2_cb_size=25778528,
    partIdx=partIdx@entry=0) at libavcodec/hevc.c:1191
#2  0x00000000008489cb in hls_coding_unit (log2_cb_size=<optimized out>, y0=<optimized out>,
    x0=<optimized out>, s=0x178a9e0) at libavcodec/hevc.c:1559
#3  hls_coding_quadtree (s=s@entry=0x178a9e0, x0=x0@entry=160, y0=y0@entry=96,
    log2_cb_size=<optimized out>, cb_depth=cb_depth@entry=0) at libavcodec/hevc.c:1670
#4  0x00000000008498ba in hls_decode_entry (avctxt=<optimized out>,
    isFilterThread=<optimized out>) at libavcodec/hevc.c:1772
#5  0x0000000000a29554 in avcodec_default_execute (c=0x179da20,
    func=0x849790 <hls_decode_entry>, arg=<optimized out>, ret=<optimized out>,
    count=<optimized out>, size=4) at libavcodec/utils.c:1014
#6  0x000000000084dde4 in hls_slice_data (s=<optimized out>) at libavcodec/hevc.c:1796
#7  decode_nal_unit (length=3714,
    nal=0x17c845b "\002\001P\323\300\307&\256ui7@\016O?\002v\314O\322ʣ\f\215W2", <incomplete sequence \346>, s=0x178a9e0) at libavcodec/hevc.c:2165
#8  decode_nal_units (s=s@entry=0x178a9e0, buf=<optimized out>, length=<optimized out>)
    at libavcodec/hevc.c:2395
#9  0x000000000084e3df in hevc_decode_frame (avctx=0x179da20, data=0x1794640,
    got_output=0x7fffffffd7ec, avpkt=0x7fffffffd570) at libavcodec/hevc.c:2499
#10 0x0000000000a2b10b in avcodec_decode_video2 (avctx=0x179da20,
    picture=picture@entry=0x1794640, got_picture_ptr=got_picture_ptr@entry=0x7fffffffd7ec,
    avpkt=avpkt@entry=0x7fffffffda50) at libavcodec/utils.c:2062
#11 0x000000000046f830 in decode_video (ist=ist@entry=0x179e140, pkt=pkt@entry=0x7fffffffda50,
    got_output=got_output@entry=0x7fffffffd7ec) at ffmpeg.c:1668
#12 0x0000000000472b2f in output_packet (pkt=0x7fffffffd9f0, ist=0x179e140) at ffmpeg.c:1866
#13 process_input (file_index=<optimized out>) at ffmpeg.c:3104
#14 0x0000000000460ba0 in transcode_step () at ffmpeg.c:3200
#15 transcode () at ffmpeg.c:3252
#16 main (argc=<optimized out>, argv=<optimized out>) at ffmpeg.c:3430
(gdb) disass $pc-32,$pc+32
Dump of assembler code from 0x867770 to 0x8677b0:
   0x0000000000867770 <put_hevc_qpel_h3_8+16>:  push   %rsp
   0x0000000000867771 <put_hevc_qpel_h3_8+17>:  lea    -0x2(%rdx),%r12
   0x0000000000867775 <put_hevc_qpel_h3_8+21>:  push   %rbp
   0x0000000000867776 <put_hevc_qpel_h3_8+22>:  xor    %ebp,%ebp
   0x0000000000867778 <put_hevc_qpel_h3_8+24>:  push   %rbx
   0x0000000000867779 <put_hevc_qpel_h3_8+25>:  mov    $0x3a,%ebx
   0x000000000086777e <put_hevc_qpel_h3_8+30>:  xchg   %ax,%ax
   0x0000000000867780 <put_hevc_qpel_h3_8+32>:  xor    %esi,%esi
   0x0000000000867782 <put_hevc_qpel_h3_8+34>:  test   %r8d,%r8d
   0x0000000000867785 <put_hevc_qpel_h3_8+37>:  mov    %r12,%rdx
   0x0000000000867788 <put_hevc_qpel_h3_8+40>:  jle    0x8677eb <put_hevc_qpel_h3_8+139>
   0x000000000086778a <put_hevc_qpel_h3_8+42>:  nopw   0x0(%rax,%rax,1)
=> 0x0000000000867790 <put_hevc_qpel_h3_8+48>:  movzbl 0x2(%rdx),%r10d
   0x0000000000867795 <put_hevc_qpel_h3_8+53>:  movzbl 0x3(%rdx),%eax
   0x0000000000867799 <put_hevc_qpel_h3_8+57>:  mov    %r10d,%r11d
   0x000000000086779c <put_hevc_qpel_h3_8+60>:  shl    $0x4,%r11d
   0x00000000008677a0 <put_hevc_qpel_h3_8+64>:  add    %r11d,%r10d
   0x00000000008677a3 <put_hevc_qpel_h3_8+67>:  movzbl 0x1(%rdx),%r11d
   0x00000000008677a8 <put_hevc_qpel_h3_8+72>:  mul    %bl
   0x00000000008677aa <put_hevc_qpel_h3_8+74>:  lea    (%r11,%r11,4),%r11d
   0x00000000008677ae <put_hevc_qpel_h3_8+78>:  sub    %r11d,%r10d
End of assembler dump.
(gdb) info register
rax            0x867760 8812384
rbx            0x3a     58
rcx            0x0      0
rdx            0x9d     157
rsi            0x0      0
rdi            0x7fffffff9190   140737488327056
rbp            0x0      0x0
rsp            0x7fffffff5098   0x7fffffff5098
r8             0x20     32
r9             0x10     16
r10            0x2      2
r11            0x0      0
r12            0x9d     157
r13            0x80     128
r14            0x20     32
r15            0x7fffffff9190   140737488327056
rip            0x867790 0x867790 <put_hevc_qpel_h3_8+48>
eflags         0x10202  [ IF RF ]
cs             0x33     51
ss             0x2b     43
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x0      0

[Michael Niedermayer]

Fixed by fa6fa2162b730336fc1d6ee0d547dcc81f4afbad as well as probably some changes from antons patch v5