Opened 13 years ago

Closed 13 years ago

Last modified 11 years ago

#34 closed defect (fixed)

Crash when using bframes

Reported by: Carl Eugen Hoyos Owned by: Michael Niedermayer
Priority: important Component: ffmpeg
Version: git Keywords: asp crash SIGSEGV roundup
Cc: Blocked By:
Blocking: Reproduced by developer: yes
Analyzed by developer: no

Description

(issue 2018)
Current FFmpeg git HEAD crashes for some samples when encoding with b-frames,
To produce a sample:

ffmpeg -i fate-suite/svq3/Vertical400kbit.sorenson3.mov -s 354x364 -qscale 2 -an out.mkv
(gdb) r -i out.mkv -bf 1 out2.mkv
FFmpeg version git-N-28742-g1caa412, Copyright (c) 2000-2011 the FFmpeg developers
  built on Mar 31 2011 13:34:23 with gcc 4.4.1 [gcc-4_4-branch revision 150839]
  configuration: --enable-libtheora
  libavutil    50. 40. 0 / 50. 40. 0
  libavcodec   52.116. 0 / 52.116. 0
  libavformat  52.104. 0 / 52.104. 0
  libavdevice  52.  4. 0 / 52.  4. 0
  libavfilter   1. 76. 0 /  1. 76. 0
  libswscale    0. 13. 0 /  0. 13. 0
[matroska,webm @ 0x122a650] Estimating duration from bitrate, this may be inaccurate
Input #0, matroska,webm, from 'out.mkv':
  Metadata:
    title           : Vertical Online SV3 Demo
    CREATION_TIME   : 2001-03-20 16:17:18
    TITLE-eng       : Vertical Online SV3 Demo
    ARTIST          : Logan Kelsey
    ARTIST-eng      : Logan Kelsey
    COPYRIGHT       : © Vertical Online 2001
    COPYRIGHT-eng   : © Vertical Online 2001
    ENCODER-eng     : Sorenson Video 3
    ENCODER         : Lavf52.104.0
  Duration: 00:00:43.60, start: 0.000000, bitrate: N/A
    Stream #0.0(eng): Video: mpeg4, yuv420p, 354x364 [PAR 1:1 DAR 177:182], 30 fps, 30 tbr, 1k tbn, 30 tbc (default)
    Metadata:
      CREATION_TIME   : 2001-03-20 16:17:18
      LANGUAGE        : eng
[buffer @ 0x1230ff0] w:354 h:364 pixfmt:yuv420p
[setdar @ 0x12314d0] a:71/73
[setdar @ 0x12314d0] w:354 h:364 -> dar:71/73 sar:1/1
Output #0, matroska, to 'out2.mkv':
  Metadata:
    title           : Vertical Online SV3 Demo
    CREATION_TIME   : 2001-03-20 16:17:18
    TITLE-eng       : Vertical Online SV3 Demo
    ARTIST          : Logan Kelsey
    ARTIST-eng      : Logan Kelsey
    COPYRIGHT       : © Vertical Online 2001
    COPYRIGHT-eng   : © Vertical Online 2001
    ENCODER-eng     : Sorenson Video 3
    encoder         : Lavf52.104.0
    Stream #0.0(eng): Video: mpeg4, yuv420p, 354x364 [PAR 1:1 DAR 177:182], q=2-31, 200 kb/s, 1k tbn, 30 tbc (default)
    Metadata:
      CREATION_TIME   : 2001-03-20 16:17:18
      LANGUAGE        : eng
Stream mapping:
  Stream #0.0 -> #0.0
Press [q] to stop encoding

Program received signal SIGSEGV, Segmentation fault.
0x0000000000832b49 in sad16_sse2 (v=0x0,
    blk2=0x1415832 '\030' <repeats 71 times>, "\031\032\033\033\033\032\032\033\033\033\033\033\033\033\033", '\030' <repeats 95 times>, "\031\032\032\033\033\033\032\032\031\030\030\030\030\030\030\030\030\030\030"...,
    blk1=0x7ffff7e8fa82 '\030' <repeats 87 times>, "\031\032\033\033\033\033\032\033\033\033\033\033\033\033\032", '\030' <repeats 95 times>, "\031\032\032"..., stride=400, h=16) at libavcodec/x86/motion_est_mmx.c:96
96          __asm__ volatile(
(gdb) bt
#0  0x0000000000832b49 in sad16_sse2 (v=0x0,
    blk2=0x1415832 '\030' <repeats 71 times>, "\031\032\033\033\033\032\032\033\033\033\033\033\033\033\033", '\030' <repeats 95 times>, "\031\032\032\033\033\033\032\032\031\030\030\030\030\030\030\030\030\030\030"...,
    blk1=0x7ffff7e8fa82 '\030' <repeats 87 times>, "\031\032\033\033\033\033\032\033\033\033\033\033\033\033\032", '\030' <repeats 95 times>, "\031\032\032"..., stride=400, h=16) at libavcodec/x86/motion_est_mmx.c:96
#1  0x000000000065fb14 in mpeg4_encode_mb (s=0x1243aa0, block=0x1280a10, motion_x=<value optimized out>, motion_y=<value optimized out>) at libavcodec/mpeg4videoenc.c:670
#2  0x00000000006a17f5 in encode_mb_internal (mb_block_count=<value optimized out>, s=<value optimized out>, mb_block_height=<value optimized out>, motion_y=<value optimized out>, motion_x=<value optimized out>)
    at libavcodec/mpegvideo_enc.c:1761
#3  encode_mb (mb_block_count=<value optimized out>, s=<value optimized out>, mb_block_height=<value optimized out>, motion_y=<value optimized out>, motion_x=<value optimized out>) at libavcodec/mpegvideo_enc.c:1797
#4  encode_thread (mb_block_count=<value optimized out>, s=<value optimized out>, mb_block_height=<value optimized out>, motion_y=<value optimized out>, motion_x=<value optimized out>) at libavcodec/mpegvideo_enc.c:2619
#5  0x0000000000753bff in avcodec_default_execute (c=0x122efd0, func=0x69ed50 <encode_thread>, arg=0x7ffff7e8fa82, ret=0x190, count=16, size=1) at libavcodec/utils.c:433
#6  0x00000000006901b7 in encode_picture (picture_number=<value optimized out>, s=<value optimized out>) at libavcodec/mpegvideo_enc.c:2982
#7  MPV_encode_picture (picture_number=<value optimized out>, s=<value optimized out>) at libavcodec/mpegvideo_enc.c:1270
#8  0x00000000007549b6 in avcodec_encode_video (avctx=0x122efd0, buf=0x7ffff7f10010 "", buf_size=773336, pict=0x7fffffffca10) at libavcodec/utils.c:618
#9  0x0000000000408adb in do_video_out (frame_size=<value optimized out>, in_picture=<value optimized out>, ist=<value optimized out>, ost=<value optimized out>, s=<value optimized out>) at ffmpeg.c:1267
#10 output_packet (frame_size=<value optimized out>, in_picture=<value optimized out>, ist=<value optimized out>, ost=<value optimized out>, s=<value optimized out>) at ffmpeg.c:1689
#11 0x000000000040ca96 in transcode (nb_output_files=<value optimized out>, nb_input_files=<value optimized out>, stream_maps=<value optimized out>, nb_stream_maps=<value optimized out>, input_files=<value optimized out>,
    output_files=<value optimized out>) at ffmpeg.c:2658
#12 0x000000000040d383 in main (argc=6, argv=<value optimized out>) at ffmpeg.c:4403
(gdb) disass $pc-25 $pc+32
Dump of assembler code from 0x832b30 to 0x832b69:
0x0000000000832b30 <sad16_sse2+0>:      movslq %ecx,%rcx
0x0000000000832b33 <sad16_sse2+3>:      pxor   %xmm2,%xmm2
0x0000000000832b37 <sad16_sse2+7>:      nopw   0x0(%rax,%rax,1)
0x0000000000832b40 <sad16_sse2+16>:     movdqu (%rdx),%xmm0
0x0000000000832b44 <sad16_sse2+20>:     movdqu (%rdx,%rcx,1),%xmm1
0x0000000000832b49 <sad16_sse2+25>:     psadbw (%rsi),%xmm0
0x0000000000832b4d <sad16_sse2+29>:     psadbw (%rsi,%rcx,1),%xmm1
0x0000000000832b52 <sad16_sse2+34>:     paddw  %xmm0,%xmm2
0x0000000000832b56 <sad16_sse2+38>:     paddw  %xmm1,%xmm2
0x0000000000832b5a <sad16_sse2+42>:     lea    (%rdx,%rcx,2),%rdx
0x0000000000832b5e <sad16_sse2+46>:     lea    (%rsi,%rcx,2),%rsi
0x0000000000832b62 <sad16_sse2+50>:     sub    $0x2,%r8d
0x0000000000832b66 <sad16_sse2+54>:     jg     0x832b40 <sad16_sse2+16>
0x0000000000832b68 <sad16_sse2+56>:     movhlps %xmm2,%xmm0
(gdb) info all-registers
rax            0x1259610        19240464
rbx            0x1243aa0        19151520
rcx            0x190    400
rdx            0x7ffff7e8fa82   140737352628866
rsi            0x1415832        21059634
rdi            0x0      0
rbp            0x152    0x152
rsp            0x7ffffffefa88   0x7ffffffefa88
r8             0x10     16
r9             0x1      1
r10            0x1      1
r11            0x0      0
r12            0x1415832        21059634
r13            0x1280a10        19401232
r14            0x0      0
r15            0x10     16
rip            0x832b49 0x832b49 <sad16_sse2+25>
eflags         0x10246  [ PF ZF IF RF ]
cs             0x33     51
ss             0x2b     43
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x0      0
fctrl          0x37f    895
fstat          0x0      0
ftag           0xaaaa   43690
fiseg          0x0      0
fioff          0x0      0
foseg          0x0      0
fooff          0x0      0
fop            0x0      0
xmm0           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x18 <repeats 16 times>}, v8_int16 = {0x1818, 0x1818, 0x1818, 0x1818, 0x1818, 0x1818, 0x1818, 0x1818}, v4_int32 = {0x18181818, 0x18181818, 0x18181818,
    0x18181818}, v2_int64 = {0x1818181818181818, 0x1818181818181818}, uint128 = 0x18181818181818181818181818181818}
xmm1           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x18 <repeats 16 times>}, v8_int16 = {0x1818, 0x1818, 0x1818, 0x1818, 0x1818, 0x1818, 0x1818, 0x1818}, v4_int32 = {0x18181818, 0x18181818, 0x18181818,
    0x18181818}, v2_int64 = {0x1818181818181818, 0x1818181818181818}, uint128 = 0x18181818181818181818181818181818}
xmm2           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm3           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x1a <repeats 16 times>}, v8_int16 = {0x1a1a, 0x1a1a, 0x1a1a, 0x1a1a, 0x1a1a, 0x1a1a, 0x1a1a, 0x1a1a}, v4_int32 = {0x1a1a1a1a, 0x1a1a1a1a, 0x1a1a1a1a,
    0x1a1a1a1a}, v2_int64 = {0x1a1a1a1a1a1a1a1a, 0x1a1a1a1a1a1a1a1a}, uint128 = 0x1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a}
xmm4           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm5           {v4_float = {0x0, 0x1, 0x0, 0x0}, v2_double = {0x1, 0x0}, v16_int8 = {0xc6, 0x4b, 0x37, 0x89, 0x41, 0x0, 0xfe, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x4bc6, 0x8937, 0x41, 0x3ffe, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x89374bc6, 0x3ffe0041, 0x0, 0x0}, v2_int64 = {0x3ffe004189374bc6, 0x0}, uint128 = 0x00000000000000003ffe004189374bc6}
xmm6           {v4_float = {0x0, 0x3, 0x0, 0x0}, v2_double = {0x1fb, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0xa0, 0x8f, 0xb6, 0x7f, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0xa000, 0xb68f, 0x407f, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0xa0000000, 0x407fb68f, 0x0, 0x0}, v2_int64 = {0x407fb68fa0000000, 0x0}, uint128 = 0x0000000000000000407fb68fa0000000}
xmm7           {v4_float = {0x0, 0x1, 0x0, 0x0}, v2_double = {0x1, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x60, 0x9e, 0xa0, 0xf6, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x6000, 0xa09e, 0x3ff6, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x60000000, 0x3ff6a09e, 0x0, 0x0}, v2_int64 = {0x3ff6a09e60000000, 0x0}, uint128 = 0x00000000000000003ff6a09e60000000}
xmm8           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
    0x0, 0x80000000, 0x0, 0x0}, v2_int64 = {0x8000000000000000, 0x0}, uint128 = 0x00000000000000008000000000000000}
xmm9           {v4_float = {0x0, 0x1, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x3fe0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
    0x0, 0x3fe00000, 0x0, 0x0}, v2_int64 = {0x3fe0000000000000, 0x0}, uint128 = 0x00000000000000003fe0000000000000}
xmm10          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm11          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0xbf, 0x2c, 0x42, 0x32, 0xef, 0xfc, 0x59, 0x3e, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x2cbf, 0x3242, 0xfcef, 0x3e59, 0x0, 0x0, 0x0,
    0x0}, v4_int32 = {0x32422cbf, 0x3e59fcef, 0x0, 0x0}, v2_int64 = {0x3e59fcef32422cbf, 0x0}, uint128 = 0x00000000000000003e59fcef32422cbf}
xmm12          {v4_float = {0x0, 0x1, 0x0, 0x0}, v2_double = {0x1, 0x0}, v16_int8 = {0xfe, 0x82, 0x2b, 0x65, 0x47, 0x15, 0xf7, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x82fe, 0x652b, 0x1547, 0x3ff7, 0x0, 0x0, 0x0,
---Type <return> to continue, or q <return> to quit---
    0x0}, v4_int32 = {0x652b82fe, 0x3ff71547, 0x0, 0x0}, v2_int64 = {0x3ff71547652b82fe, 0x0}, uint128 = 0x00000000000000003ff71547652b82fe}
xmm13          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0xdc, 0xe8, 0x34, 0x76, 0xa6, 0x4b, 0x20, 0x3c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0xe8dc, 0x7634, 0x4ba6, 0x3c20, 0x0, 0x0, 0x0,
    0x0}, v4_int32 = {0x7634e8dc, 0x3c204ba6, 0x0, 0x0}, v2_int64 = {0x3c204ba67634e8dc, 0x0}, uint128 = 0x00000000000000003c204ba67634e8dc}
xmm14          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x90, 0x34, 0x37, 0xb9, 0x3d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x9000, 0x3734, 0x3db9, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x90000000, 0x3db93734, 0x0, 0x0}, v2_int64 = {0x3db9373490000000, 0x0}, uint128 = 0x00000000000000003db9373490000000}
xmm15          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]

Change History (2)

comment:1 by Michael Niedermayer, 13 years ago

Resolution: fixed
Status: newclosed

comment:2 by Carl Eugen Hoyos, 11 years ago

Keywords: asp crash SIGSEGV roundup added
Note: See TracTickets for help on using tickets.