Opened 6 years ago

Closed 6 years ago

#7078 closed defect (fixed)

deshake crashes with crop and sse2

Reported by: Carl Eugen Hoyos Owned by:
Priority: important Component: avfilter
Version: git-master Keywords: deshake crash SIGSEGV
Cc: Blocked By:
Blocking: Reproduced by developer: no
Analyzed by developer: no

Description

Alexander found this crash:

(gdb) r -lavfi testsrc2=s=hd1080,crop=1720:1080,deshake -f null -
Starting program: ffmpeg_g -lavfi testsrc2=s=hd1080,crop=1720:1080,deshake -f null -
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
ffmpeg version N-90284-g950170b Copyright (c) 2000-2018 the FFmpeg developers
  built with gcc 4.8 (SUSE Linux)
  configuration: --enable-gpl --enable-libx264 --enable-libx265 --enable-libvpx --enable-gnutls --enable-libxml2
  libavutil      56.  9.100 / 56.  9.100
  libavcodec     58. 14.100 / 58. 14.100
  libavformat    58. 10.100 / 58. 10.100
  libavdevice    58.  2.100 / 58.  2.100
  libavfilter     7. 12.100 /  7. 12.100
  libswscale      5.  0.102 /  5.  0.102
  libswresample   3.  0.101 /  3.  0.101
  libpostproc    55.  0.100 / 55.  0.100
Stream mapping:
  deshake -> Stream #0:0 (wrapped_avframe)
Press [q] to stop, [?] for help
[New Thread 0x7ffff2980700 (LWP 9116)]
[New Thread 0x7ffff217f700 (LWP 9117)]
Output #0, null, to 'pipe:':
  Metadata:
    encoder         : Lavf58.10.100
    Stream #0:0: Video: wrapped_avframe, yuv420p, 1720x1080 [SAR 1:1 DAR 43:27], q=2-31, 200 kb/s, 25 fps, 25 tbn, 25 tbc (default)
    Metadata:
      encoder         : Lavc58.14.100 wrapped_avframe

Program received signal SIGSEGV, Segmentation fault.
0x0000000001072964 in ff_pixelutils_sad_u_16x16_sse2 ()
(gdb) bt
#0  0x0000000001072964 in ff_pixelutils_sad_u_16x16_sse2 ()
#1  0x0000000000507b62 in find_block_motion (mv=<synthetic pointer>, stride=1920, cy=16, cx=864, 
    src2=0x7ffff17800a4 'Q' <repeats 200 times>..., src1=0x7ffff17800a4 'Q' <repeats 200 times>..., 
    deshake=0x1fb11c0) at libavfilter/vf_deshake.c:136
#2  find_motion (deshake=deshake@entry=0x1fb11c0, src1=0x7ffff17800a4 'Q' <repeats 200 times>..., 
    src2=0x7ffff17800a4 'Q' <repeats 200 times>..., width=1720, height=1080, stride=1920, t=t@entry=0x7fffffffd200)
    at libavfilter/vf_deshake.c:266
#3  0x00000000005083ef in filter_frame (link=link@entry=0x1fb0180, in=0x1fc1700) at libavfilter/vf_deshake.c:454
#4  0x00000000004c5745 in ff_filter_frame_framed (frame=0x1fc1700, link=0x1fb0180) at libavfilter/avfilter.c:1115
#5  ff_filter_frame_to_filter (link=0x1fb0180) at libavfilter/avfilter.c:1263
#6  ff_filter_activate_default (filter=<optimized out>) at libavfilter/avfilter.c:1312
#7  ff_filter_activate (filter=<optimized out>) at libavfilter/avfilter.c:1473
#8  0x00000000004c8e4c in ff_filter_graph_run_once (graph=<optimized out>) at libavfilter/avfiltergraph.c:1453
#9  0x00000000004c94f1 in get_frame_internal (samples=0, flags=1, frame=0x0, ctx=0x1faf280)
    at libavfilter/buffersink.c:110
#10 av_buffersink_get_frame_flags (ctx=0x1faf280, frame=frame@entry=0x0, flags=flags@entry=1)
    at libavfilter/buffersink.c:121
#11 0x00000000004c8ba9 in avfilter_graph_request_oldest (graph=0x1faf1c0) at libavfilter/avfiltergraph.c:1406
#12 0x000000000048675e in transcode_from_filter (best_ist=<synthetic pointer>, graph=0x1facc00)
    at fftools/ffmpeg.c:4490
#13 transcode_step () at fftools/ffmpeg.c:4565
#14 transcode () at fftools/ffmpeg.c:4641
#15 main (argc=<optimized out>, argv=<optimized out>) at fftools/ffmpeg.c:4844
(gdb) disass $pc-4,$pc+32
Dump of assembler code from 0x1072960 to 0x1072984:
   0x0000000001072960 <ff_pixelutils_sad_u_16x16_sse2+0>:       movdqu (%rdx),%xmm2
=> 0x0000000001072964 <ff_pixelutils_sad_u_16x16_sse2+4>:       psadbw (%rdi),%xmm2
   0x0000000001072968 <ff_pixelutils_sad_u_16x16_sse2+8>:       movdqu (%rdx,%rcx,1),%xmm1
   0x000000000107296d <ff_pixelutils_sad_u_16x16_sse2+13>:      psadbw (%rdi,%rsi,1),%xmm1
   0x0000000001072972 <ff_pixelutils_sad_u_16x16_sse2+18>:      paddw  %xmm1,%xmm2
   0x0000000001072976 <ff_pixelutils_sad_u_16x16_sse2+22>:      lea    (%rdi,%rsi,2),%rdi
   0x000000000107297a <ff_pixelutils_sad_u_16x16_sse2+26>:      lea    (%rdx,%rcx,2),%rdx
   0x000000000107297e <ff_pixelutils_sad_u_16x16_sse2+30>:      movdqu (%rdx),%xmm0
   0x0000000001072982 <ff_pixelutils_sad_u_16x16_sse2+34>:      psadbw (%rdi),%xmm0
End of assembler dump.
(gdb) info all-register
rax            0xfffffffffffffff0       -16
rbx            0x780    1920
rcx            0x780    1920
rdx            0x7ffff178f414   140737244623892
rsi            0x780    1920
rdi            0x7ffff1787c04   140737244593156
rbp            0x0      0x0
rsp            0x7fffffffd0a8   0x7fffffffd0a8
r8             0x357    855
r9             0x9      9
r10            0x351    849
r11            0x352    850
r12            0x1fb11c0        33231296
r13            0xfffffff0       4294967280
r14            0x0      0
r15            0xfffffff0       4294967280
rip            0x1072964        0x1072964 <ff_pixelutils_sad_u_16x16_sse2+4>
eflags         0x10206  [ PF IF RF ]
cs             0x33     51
ss             0x2b     43
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x0      0
st0            0        (raw 0x00000000000000000000)
st1            0        (raw 0x00000000000000000000)
st2            0        (raw 0x00000000000000000000)
st3            0        (raw 0x00000000000000000000)
st4            0        (raw 0x00000000000000000000)
st5            0        (raw 0x00000000000000000000)
st6            0        (raw 0x00000000000000000000)
st7            0        (raw 0x00000000000000000000)
fctrl          0x37f    895
fstat          0x0      0
ftag           0xffff   65535
fiseg          0x0      0
fioff          0x0      0
foseg          0x0      0
fooff          0x0      0
fop            0x0      0
xmm0           {v4_float = {0x0, 0x1, 0x0, 0x1}, v2_double = {0x0, 0x0}, v16_int8 = {0x9a, 0x99, 0x99, 0x99, 0x99, 
    0x99, 0xb9, 0x3f, 0x9a, 0x99, 0x99, 0x99, 0x99, 0x99, 0xb9, 0x3f}, v8_int16 = {0x999a, 0x9999, 0x9999, 0x3fb9, 
    0x999a, 0x9999, 0x9999, 0x3fb9}, v4_int32 = {0x9999999a, 0x3fb99999, 0x9999999a, 0x3fb99999}, v2_int64 = {
    0x3fb999999999999a, 0x3fb999999999999a}, uint128 = 0x3fb999999999999a3fb999999999999a}
xmm1           {v4_float = {0x0, 0x2, 0x0, 0x0}, v2_double = {0x14, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 
    0x0, 0x34, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x4034, 0x0, 0x0, 0x0, 
    0x0}, v4_int32 = {0x0, 0x40340000, 0x0, 0x0}, v2_int64 = {0x4034000000000000, 0x0}, 
  uint128 = 0x00000000000000004034000000000000}
xmm2           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x29 <repeats 16 times>}, 
  v8_int16 = {0x2929, 0x2929, 0x2929, 0x2929, 0x2929, 0x2929, 0x2929, 0x2929}, v4_int32 = {0x29292929, 0x29292929, 
    0x29292929, 0x29292929}, v2_int64 = {0x2929292929292929, 0x2929292929292929}, 
  uint128 = 0x29292929292929292929292929292929}
xmm3           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
    0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0xff00, 0x0}, 
  v4_int32 = {0x0, 0xff0000, 0x0, 0xff00}, v2_int64 = {0xff000000000000, 0xff0000000000}, 
  uint128 = 0x0000ff000000000000ff000000000000}
xmm4           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x8000000000000000}, v16_int8 = {0x2c, 0x20, 
    0x71, 0x3d, 0x32, 0x2d, 0x33, 0x31, 0x2c, 0x20, 0x32, 0x30, 0x30, 0x20, 0x6b, 0x62}, v8_int16 = {0x202c, 
    0x3d71, 0x2d32, 0x3133, 0x202c, 0x3032, 0x2030, 0x626b}, v4_int32 = {0x3d71202c, 0x31332d32, 0x3032202c, 
    0x626b2030}, v2_int64 = {0x31332d323d71202c, 0x626b20303032202c}, uint128 = 0x626b20303032202c31332d323d71202c}
xmm5           {v4_float = {0x0, 0xffffffff, 0x3, 0x3}, v2_double = {0xffffffffffffffff, 0x20}, v16_int8 = {0x0, 
    0x0, 0x0, 0x0, 0x0, 0x0, 0xf0, 0xbf, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40}, v8_int16 = {0x0, 0x0, 
    0x0, 0xbff0, 0x4040, 0x4040, 0x4040, 0x4040}, v4_int32 = {0x0, 0xbff00000, 0x40404040, 0x40404040}, 
  v2_int64 = {0xbff0000000000000, 0x4040404040404040}, uint128 = 0x4040404040404040bff0000000000000}
xmm6           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0xcd, 0xcc, 0xcc, 0x3d, 0xcd, 
    0xcc, 0xcc, 0x3d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0xcccd, 0x3dcc, 0xcccd, 0x3dcc, 0x0, 
    0x0, 0x0, 0x0}, v4_int32 = {0x3dcccccd, 0x3dcccccd, 0x0, 0x0}, v2_int64 = {0x3dcccccd3dcccccd, 0x0}, 
  uint128 = 0x00000000000000003dcccccd3dcccccd}
xmm7           {v4_float = {0x3, 0x3, 0x0, 0x0}, v2_double = {0x20, 0x0}, v16_int8 = {0x0, 0x0, 0x40, 0x40, 0x0, 
    0x0, 0x40, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x4040, 0x0, 0x4040, 0x0, 0x0, 0x0, 
    0x0}, v4_int32 = {0x40400000, 0x40400000, 0x0, 0x0}, v2_int64 = {0x4040000040400000, 0x0}, 
  uint128 = 0x00000000000000004040000040400000}
xmm8           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, 
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, 
  uint128 = 0x00000000000000000000000000000000}
xmm9           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, 
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, 
  uint128 = 0x00000000000000000000000000000000}
xmm10          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, 
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, 
  uint128 = 0x00000000000000000000000000000000}
xmm11          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, 
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, 
  uint128 = 0x00000000000000000000000000000000}
xmm12          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0xff, 
    0x0 <repeats 14 times>}, v8_int16 = {0xff00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0xff00, 0x0, 0x0, 
    0x0}, v2_int64 = {0xff00, 0x0}, uint128 = 0x0000000000000000000000000000ff00}
xmm13          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, 
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, 
  uint128 = 0x00000000000000000000000000000000}
xmm14          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, 
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, 
  uint128 = 0x00000000000000000000000000000000}
xmm15          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, 
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, 
  uint128 = 0x00000000000000000000000000000000}
mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]

Change History (1)

comment:1 by Carl Eugen Hoyos, 6 years ago

Resolution: fixed
Status: newclosed
Note: See TracTickets for help on using tickets.