Ticket #799: x264_predict_8x8_filter_ssse3() - gdb.txt

File x264_predict_8x8_filter_ssse3() - gdb.txt, 9.8 KB (added by treestump, 5 years ago)

gdb output

Line 
1C:\Documents and Settings\foobar\workspace\ffmpeg_debugging\Debug>gdb ffmpeg_debugging.exe
2GNU gdb (GDB) 7.3.1
3Copyright (C) 2011 Free Software Foundation, Inc.
4License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
5This is free software: you are free to change and redistribute it.
6There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
7and "show warranty" for details.
8This GDB was configured as "mingw32".
9For bug reporting instructions, please see:
10<http://www.gnu.org/software/gdb/bugs/>...
11Reading symbols from c:\documents and settings\foobar\workspace\ffmpeg_debugging\debug\ffmpeg_debugging.exe...done.
12(gdb) r
13Starting program: c:\documents and settings\foobar\workspace\ffmpeg_debugging\debug\ffmpeg_debugging.exe
14[New Thread 1388.0x1a4]
15[libx264 @ 003e6700] interlace + weightp is not implemented
16[libx264 @ 003e6700] using cpu capabilities: MMX2 SSE2Fast SSSE3 Cache64
17[New Thread 1388.0x444]
18[New Thread 1388.0x6a4]
19[New Thread 1388.0x894]
20[New Thread 1388.0xb54]
21[libx264 @ 003e6700] profile High, level 4.0
22[libx264 @ 003e6700] 264 - core 120 - H.264/MPEG-4 AVC codec - Copyleft 2003-2011 - http://www.videolan.org/x264.html - options: c
23abac=1 ref=4 deblock=1:0:0 analyse=0x3:0x113 me=umh subme=8 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=0 trellis=1 8
24x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=3 sliced_threads=0 nr=0 decimate=1 interlaced=tff bluray_com
25pat=0 constrained_intra=0 bframes=3 b_pyramid=0 b_adapt=1 b_bias=0 direct=3 weightb=1 open_gop=0 weightp=0 keyint=250 keyint_min=2
265 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=22.0 qcomp=0.60 qpmin=10 qpmax=51 qpstep=4 ip_ratio=1.41 aq=1:1.
2700
28Frame: 0
29Frame: 1
30Frame: 2
31Frame: 3
32Frame: 4
33Frame: 5
34Frame: 6
35Frame: 7
36Frame: 8
37Frame: 9
38Frame: 10
39Frame: 11
40Frame: 12
41Frame: 13
42Frame: 14
43Frame: 15
44Frame: 16
45Frame: 17
46Frame: 18
47Frame: 19
48Frame: 20
49Frame: 21
50Frame: 22
51Frame: 23
52Frame: 24
53Frame: 25
54Frame: 26
55Frame: 27
56Frame: 28
57Frame: 29
58Frame: 30
59Frame: 31
60Frame: 32
61Frame: 33
62Frame: 34
63Frame: 35
64Frame: 36
65Frame: 37
66Frame: 38
67Frame: 39
68Frame: 40
69Frame: 41
70Frame: 42
71Frame: 43
72Frame: 44
73Frame: 45
74
75Program received signal SIGSEGV, Segmentation fault.
76[Switching to Thread 1388.0x894]
770x00734b0f in x264_predict_8x8_filter_ssse3 ()
78(gdb) bt
79#0  0x00734b0f in x264_predict_8x8_filter_ssse3 ()
80#1  0x00702ee1 in _fu7756____stack_chk_guard () at encoder/analyse.c:1260
81#2  0x00708d86 in _fu7782____stack_chk_guard () at encoder/analyse.c:3018
82#3  0x006a8995 in _fu6832____stack_chk_guard () at encoder/encoder.c:2168
83#4  0x006bbe92 in x264_stack_align ()
84#5  0x006a3503 in _fu6814____stack_chk_guard () at encoder/encoder.c:2469
85#6  0x006d11dd in _fu7486____stack_chk_guard () at common/threadpool.c:69
86#7  0x0072179c in _fu8076____stack_chk_guard () at common/win32thread.c:65
87#8  0x77c3a3b0 in msvcrt!_endthreadex () from C:\WINDOWS\system32\msvcrt.dll
88#9  0x7c80b729 in KERNEL32!GetModuleFileNameA () from C:\WINDOWS\system32\kernel32.dll
89#10 0x00000000 in ?? ()
90(gdb) disass $pc-32,$pc+32
91Dump of assembler code from 0x734aef to 0x734b2f:
92   0x00734aef <x264_predict_8x8_filter_ssse3+95>:       palignr $0x7,%mm0,%mm4
93   0x00734af4 <x264_predict_8x8_filter_ssse3+100>:      palignr $0x1,%mm3,%mm1
94   0x00734af9 <x264_predict_8x8_filter_ssse3+105>:      movq   %mm1,%mm5
95   0x00734afc <x264_predict_8x8_filter_ssse3+108>:      pavgb  %mm4,%mm1
96   0x00734aff <x264_predict_8x8_filter_ssse3+111>:      pxor   %mm5,%mm4
97   0x00734b02 <x264_predict_8x8_filter_ssse3+114>:      pand   0x7cafb0,%mm4
98   0x00734b09 <x264_predict_8x8_filter_ssse3+121>:      psubusb %mm4,%mm1
99   0x00734b0c <x264_predict_8x8_filter_ssse3+124>:      pavgb  %mm1,%mm3
100=> 0x00734b0f <x264_predict_8x8_filter_ssse3+127>:      movq   %mm3,0x8(%esi)
101   0x00734b13 <x264_predict_8x8_filter_ssse3+131>:      movzbl 0x87(%eax),%ecx
102   0x00734b1a <x264_predict_8x8_filter_ssse3+138>:      movzbl 0x67(%eax),%edi
103   0x00734b1e <x264_predict_8x8_filter_ssse3+142>:      lea    0x2(%ecx,%ecx,2),%ecx
104   0x00734b22 <x264_predict_8x8_filter_ssse3+146>:      add    %edi,%ecx
105   0x00734b24 <x264_predict_8x8_filter_ssse3+148>:      shr    $0x2,%ecx
106   0x00734b27 <x264_predict_8x8_filter_ssse3+151>:      mov    %cl,0x7(%esi)
107   0x00734b2a <x264_predict_8x8_filter_ssse3+154>:      mov    %cl,0x6(%esi)
108   0x00734b2d <x264_predict_8x8_filter_ssse3+157>:      test   $0x2,%bl
109End of assembler dump.
110(gdb) info all-registers
111eax            0x18d54c0        26039488
112ecx            0x0      0
113edx            0x1      1
114ebx            0xf      15
115esp            0x245bfd0        0x245bfd0
116ebp            0x18d05f0        0x18d05f0
117esi            0x0      0
118edi            0x1      1
119eip            0x734b0f 0x734b0f <x264_predict_8x8_filter_ssse3+127>
120eflags         0x10246  [ PF ZF IF RF ]
121cs             0x1b     27
122ss             0x23     35
123ds             0x23     35
124es             0x23     35
125fs             0x3b     59
126gs             0x0      0
127st0            -nan(0x1010101010101010) (raw 0xffff1010101010101010)
128st1            -nan(0xe10101010101010)  (raw 0xffff0e10101010101010)
129st2            -nan(0x1010101010101010) (raw 0xffff1010101010101010)
130st3            -nan(0xf10101010101010)  (raw 0xffff0f10101010101010)
131st4            -nan(0x100000000000000)  (raw 0xffff0100000000000000)
132st5            -nan(0xd10101010101010)  (raw 0xffff0d10101010101010)
133st6            -inf     (raw 0xffff0000000000000000)
134st7            -inf     (raw 0xffff0000000000000000)
135fctrl          0xffff027f       -64897
136fstat          0xffff0000       -65536
137ftag           0xffffaaaa       -21846
138fiseg          0x0      0
139fioff          0x0      0
140foseg          0xffff0000       -65536
141fooff          0x0      0
142fop            0x0      0
143xmm0           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x8000000000000000, 0x8000000000000000}, v16_int8 = {
144    0xff <repeats 16 times>}, v8_int16 = {0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff}, v4_int32 = {
145    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff}, v2_int64 = {0xffffffffffffffff, 0xffffffffffffffff},
146  uint128 = 0xffffffffffffffffffffffffffffffff}
147xmm1           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0,
148    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
149  uint128 = 0x00000000000000000000000000000000}
150xmm2           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0,
151    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
152  uint128 = 0x00000000000000000000000000000000}
153xmm3           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x39, 0x0, 0x7c, 0x5, 0x39, 0x0, 0x7c,
154    0x5, 0x40, 0xac, 0xb5, 0x5, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x39, 0x57c, 0x39, 0x57c, 0xac40, 0x5b5, 0x0, 0x0},
155  v4_int32 = {0x57c0039, 0x57c0039, 0x5b5ac40, 0x0}, v2_int64 = {0x57c0039057c0039, 0x5b5ac40},
156  uint128 = 0x0000000005b5ac40057c0039057c0039}
157xmm4           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
158    0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0}, v4_int32 = {0x1, 0x0, 0x2,
159    0x0}, v2_int64 = {0x1, 0x2}, uint128 = 0x00000000000000020000000000000001}
160xmm5           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10,
161    0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0}, v8_int16 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}, v4_int32 = {
162    0x100010, 0x100010, 0x100010, 0x100010}, v2_int64 = {0x10001000100010, 0x10001000100010},
163  uint128 = 0x00100010001000100010001000100010}
164xmm6           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10,
165    0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0}, v8_int16 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}, v4_int32 = {
166    0x100010, 0x100010, 0x100010, 0x100010}, v2_int64 = {0x10001000100010, 0x10001000100010},
167  uint128 = 0x00100010001000100010001000100010}
168xmm7           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0xff, 0x0, 0xff, 0x0, 0xff, 0x0, 0xff,
169    0x0, 0xff, 0x0, 0xff, 0x0, 0xff, 0x0, 0xff, 0x0}, v8_int16 = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, v4_int32 = {
170    0xff00ff, 0xff00ff, 0xff00ff, 0xff00ff}, v2_int64 = {0xff00ff00ff00ff, 0xff00ff00ff00ff},
171  uint128 = 0x00ff00ff00ff00ff00ff00ff00ff00ff}
172mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]
173mm0            {uint64 = 0x1010101010101010, v2_int32 = {0x10101010, 0x10101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0x1010},
174  v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}}
175mm1            {uint64 = 0xe10101010101010, v2_int32 = {0x10101010, 0xe101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xe10},
176  v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xe}}
177mm2            {uint64 = 0x1010101010101010, v2_int32 = {0x10101010, 0x10101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0x1010},
178  v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}}
179mm3            {uint64 = 0xf10101010101010, v2_int32 = {0x10101010, 0xf101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xf10},
180  v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xf}}
181mm4            {uint64 = 0x100000000000000, v2_int32 = {0x0, 0x1000000}, v4_int16 = {0x0, 0x0, 0x0, 0x100}, v8_int8 = {0x0,
182    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}
183mm5            {uint64 = 0xd10101010101010, v2_int32 = {0x10101010, 0xd101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xd10},
184  v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xd}}
185mm6            {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0, 0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
186    0x0, 0x0}}
187mm7            {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0, 0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
188    0x0, 0x0}}
189(gdb)