Ticket #799: x264_predict_8x8_filter_ssse3() - gdb.txt

File x264_predict_8x8_filter_ssse3() - gdb.txt, 9.8 KB (added by treestump, 15 years ago)

gdb output

Line 
1C:\Documents and Settings\foobar\workspace\ffmpeg_debugging\Debug>gdb ffmpeg_debugging.exe
2GNU gdb (GDB) 7.3.1
3Copyright (C) 2011 Free Software Foundation, Inc.
4License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
5This is free software: you are free to change and redistribute it.
6There is NO WARRANTY, to the extent permitted by law. Type "show copying"
7and "show warranty" for details.
8This GDB was configured as "mingw32".
9For bug reporting instructions, please see:
10<http://www.gnu.org/software/gdb/bugs/>...
11Reading symbols from c:\documents and settings\foobar\workspace\ffmpeg_debugging\debug\ffmpeg_debugging.exe...done.
12(gdb) r
13Starting program: c:\documents and settings\foobar\workspace\ffmpeg_debugging\debug\ffmpeg_debugging.exe
14[New Thread 1388.0x1a4]
15[libx264 @ 003e6700] interlace + weightp is not implemented
16[libx264 @ 003e6700] using cpu capabilities: MMX2 SSE2Fast SSSE3 Cache64
17[New Thread 1388.0x444]
18[New Thread 1388.0x6a4]
19[New Thread 1388.0x894]
20[New Thread 1388.0xb54]
21[libx264 @ 003e6700] profile High, level 4.0
22[libx264 @ 003e6700] 264 - core 120 - H.264/MPEG-4 AVC codec - Copyleft 2003-2011 - http://www.videolan.org/x264.html - options: c
23abac=1 ref=4 deblock=1:0:0 analyse=0x3:0x113 me=umh subme=8 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=0 trellis=1 8
24x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=3 sliced_threads=0 nr=0 decimate=1 interlaced=tff bluray_com
25pat=0 constrained_intra=0 bframes=3 b_pyramid=0 b_adapt=1 b_bias=0 direct=3 weightb=1 open_gop=0 weightp=0 keyint=250 keyint_min=2
265 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=22.0 qcomp=0.60 qpmin=10 qpmax=51 qpstep=4 ip_ratio=1.41 aq=1:1.
2700
28Frame: 0
29Frame: 1
30Frame: 2
31Frame: 3
32Frame: 4
33Frame: 5
34Frame: 6
35Frame: 7
36Frame: 8
37Frame: 9
38Frame: 10
39Frame: 11
40Frame: 12
41Frame: 13
42Frame: 14
43Frame: 15
44Frame: 16
45Frame: 17
46Frame: 18
47Frame: 19
48Frame: 20
49Frame: 21
50Frame: 22
51Frame: 23
52Frame: 24
53Frame: 25
54Frame: 26
55Frame: 27
56Frame: 28
57Frame: 29
58Frame: 30
59Frame: 31
60Frame: 32
61Frame: 33
62Frame: 34
63Frame: 35
64Frame: 36
65Frame: 37
66Frame: 38
67Frame: 39
68Frame: 40
69Frame: 41
70Frame: 42
71Frame: 43
72Frame: 44
73Frame: 45
74
75Program received signal SIGSEGV, Segmentation fault.
76[Switching to Thread 1388.0x894]
770x00734b0f in x264_predict_8x8_filter_ssse3 ()
78(gdb) bt
79#0 0x00734b0f in x264_predict_8x8_filter_ssse3 ()
80#1 0x00702ee1 in _fu7756____stack_chk_guard () at encoder/analyse.c:1260
81#2 0x00708d86 in _fu7782____stack_chk_guard () at encoder/analyse.c:3018
82#3 0x006a8995 in _fu6832____stack_chk_guard () at encoder/encoder.c:2168
83#4 0x006bbe92 in x264_stack_align ()
84#5 0x006a3503 in _fu6814____stack_chk_guard () at encoder/encoder.c:2469
85#6 0x006d11dd in _fu7486____stack_chk_guard () at common/threadpool.c:69
86#7 0x0072179c in _fu8076____stack_chk_guard () at common/win32thread.c:65
87#8 0x77c3a3b0 in msvcrt!_endthreadex () from C:\WINDOWS\system32\msvcrt.dll
88#9 0x7c80b729 in KERNEL32!GetModuleFileNameA () from C:\WINDOWS\system32\kernel32.dll
89#10 0x00000000 in ?? ()
90(gdb) disass $pc-32,$pc+32
91Dump of assembler code from 0x734aef to 0x734b2f:
92 0x00734aef <x264_predict_8x8_filter_ssse3+95>: palignr $0x7,%mm0,%mm4
93 0x00734af4 <x264_predict_8x8_filter_ssse3+100>: palignr $0x1,%mm3,%mm1
94 0x00734af9 <x264_predict_8x8_filter_ssse3+105>: movq %mm1,%mm5
95 0x00734afc <x264_predict_8x8_filter_ssse3+108>: pavgb %mm4,%mm1
96 0x00734aff <x264_predict_8x8_filter_ssse3+111>: pxor %mm5,%mm4
97 0x00734b02 <x264_predict_8x8_filter_ssse3+114>: pand 0x7cafb0,%mm4
98 0x00734b09 <x264_predict_8x8_filter_ssse3+121>: psubusb %mm4,%mm1
99 0x00734b0c <x264_predict_8x8_filter_ssse3+124>: pavgb %mm1,%mm3
100=> 0x00734b0f <x264_predict_8x8_filter_ssse3+127>: movq %mm3,0x8(%esi)
101 0x00734b13 <x264_predict_8x8_filter_ssse3+131>: movzbl 0x87(%eax),%ecx
102 0x00734b1a <x264_predict_8x8_filter_ssse3+138>: movzbl 0x67(%eax),%edi
103 0x00734b1e <x264_predict_8x8_filter_ssse3+142>: lea 0x2(%ecx,%ecx,2),%ecx
104 0x00734b22 <x264_predict_8x8_filter_ssse3+146>: add %edi,%ecx
105 0x00734b24 <x264_predict_8x8_filter_ssse3+148>: shr $0x2,%ecx
106 0x00734b27 <x264_predict_8x8_filter_ssse3+151>: mov %cl,0x7(%esi)
107 0x00734b2a <x264_predict_8x8_filter_ssse3+154>: mov %cl,0x6(%esi)
108 0x00734b2d <x264_predict_8x8_filter_ssse3+157>: test $0x2,%bl
109End of assembler dump.
110(gdb) info all-registers
111eax 0x18d54c0 26039488
112ecx 0x0 0
113edx 0x1 1
114ebx 0xf 15
115esp 0x245bfd0 0x245bfd0
116ebp 0x18d05f0 0x18d05f0
117esi 0x0 0
118edi 0x1 1
119eip 0x734b0f 0x734b0f <x264_predict_8x8_filter_ssse3+127>
120eflags 0x10246 [ PF ZF IF RF ]
121cs 0x1b 27
122ss 0x23 35
123ds 0x23 35
124es 0x23 35
125fs 0x3b 59
126gs 0x0 0
127st0 -nan(0x1010101010101010) (raw 0xffff1010101010101010)
128st1 -nan(0xe10101010101010) (raw 0xffff0e10101010101010)
129st2 -nan(0x1010101010101010) (raw 0xffff1010101010101010)
130st3 -nan(0xf10101010101010) (raw 0xffff0f10101010101010)
131st4 -nan(0x100000000000000) (raw 0xffff0100000000000000)
132st5 -nan(0xd10101010101010) (raw 0xffff0d10101010101010)
133st6 -inf (raw 0xffff0000000000000000)
134st7 -inf (raw 0xffff0000000000000000)
135fctrl 0xffff027f -64897
136fstat 0xffff0000 -65536
137ftag 0xffffaaaa -21846
138fiseg 0x0 0
139fioff 0x0 0
140foseg 0xffff0000 -65536
141fooff 0x0 0
142fop 0x0 0
143xmm0 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x8000000000000000, 0x8000000000000000}, v16_int8 = {
144 0xff <repeats 16 times>}, v8_int16 = {0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff}, v4_int32 = {
145 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff}, v2_int64 = {0xffffffffffffffff, 0xffffffffffffffff},
146 uint128 = 0xffffffffffffffffffffffffffffffff}
147xmm1 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0,
148 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
149 uint128 = 0x00000000000000000000000000000000}
150xmm2 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0,
151 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
152 uint128 = 0x00000000000000000000000000000000}
153xmm3 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x39, 0x0, 0x7c, 0x5, 0x39, 0x0, 0x7c,
154 0x5, 0x40, 0xac, 0xb5, 0x5, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x39, 0x57c, 0x39, 0x57c, 0xac40, 0x5b5, 0x0, 0x0},
155 v4_int32 = {0x57c0039, 0x57c0039, 0x5b5ac40, 0x0}, v2_int64 = {0x57c0039057c0039, 0x5b5ac40},
156 uint128 = 0x0000000005b5ac40057c0039057c0039}
157xmm4 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
158 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0}, v4_int32 = {0x1, 0x0, 0x2,
159 0x0}, v2_int64 = {0x1, 0x2}, uint128 = 0x00000000000000020000000000000001}
160xmm5 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10,
161 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0}, v8_int16 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}, v4_int32 = {
162 0x100010, 0x100010, 0x100010, 0x100010}, v2_int64 = {0x10001000100010, 0x10001000100010},
163 uint128 = 0x00100010001000100010001000100010}
164xmm6 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10,
165 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0}, v8_int16 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}, v4_int32 = {
166 0x100010, 0x100010, 0x100010, 0x100010}, v2_int64 = {0x10001000100010, 0x10001000100010},
167 uint128 = 0x00100010001000100010001000100010}
168xmm7 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0xff, 0x0, 0xff, 0x0, 0xff, 0x0, 0xff,
169 0x0, 0xff, 0x0, 0xff, 0x0, 0xff, 0x0, 0xff, 0x0}, v8_int16 = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, v4_int32 = {
170 0xff00ff, 0xff00ff, 0xff00ff, 0xff00ff}, v2_int64 = {0xff00ff00ff00ff, 0xff00ff00ff00ff},
171 uint128 = 0x00ff00ff00ff00ff00ff00ff00ff00ff}
172mxcsr 0x1fa0 [ PE IM DM ZM OM UM PM ]
173mm0 {uint64 = 0x1010101010101010, v2_int32 = {0x10101010, 0x10101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0x1010},
174 v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}}
175mm1 {uint64 = 0xe10101010101010, v2_int32 = {0x10101010, 0xe101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xe10},
176 v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xe}}
177mm2 {uint64 = 0x1010101010101010, v2_int32 = {0x10101010, 0x10101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0x1010},
178 v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}}
179mm3 {uint64 = 0xf10101010101010, v2_int32 = {0x10101010, 0xf101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xf10},
180 v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xf}}
181mm4 {uint64 = 0x100000000000000, v2_int32 = {0x0, 0x1000000}, v4_int16 = {0x0, 0x0, 0x0, 0x100}, v8_int8 = {0x0,
182 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}
183mm5 {uint64 = 0xd10101010101010, v2_int32 = {0x10101010, 0xd101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xd10},
184 v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xd}}
185mm6 {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0, 0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
186 0x0, 0x0}}
187mm7 {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0, 0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
188 0x0, 0x0}}
189(gdb)