40 #define C0 23170 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
41 #define C1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
42 #define C2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
43 #define C3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
44 #define C4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5
45 #define C5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
46 #define C6 8867 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
47 #define C7 4520 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
50 #define COL_SHIFT 20 // 6
86 int16_t *
const temp= (int16_t*)align_tmp;
89 #if 0 //Alternative, simpler variant
91 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
92 "movq " #src0 ", %%mm0 \n\t" \
93 "movq " #src4 ", %%mm1 \n\t" \
94 "movq " #src1 ", %%mm2 \n\t" \
95 "movq " #src5 ", %%mm3 \n\t" \
96 "movq 16(%2), %%mm4 \n\t" \
97 "pmaddwd %%mm0, %%mm4 \n\t" \
98 "movq 24(%2), %%mm5 \n\t" \
99 "pmaddwd %%mm5, %%mm0 \n\t" \
100 "movq 32(%2), %%mm5 \n\t" \
101 "pmaddwd %%mm1, %%mm5 \n\t" \
102 "movq 40(%2), %%mm6 \n\t" \
103 "pmaddwd %%mm6, %%mm1 \n\t" \
104 "movq 48(%2), %%mm7 \n\t" \
105 "pmaddwd %%mm2, %%mm7 \n\t" \
106 #rounder ", %%mm4 \n\t"\
107 "movq %%mm4, %%mm6 \n\t" \
108 "paddd %%mm5, %%mm4 \n\t" \
109 "psubd %%mm5, %%mm6 \n\t" \
110 "movq 56(%2), %%mm5 \n\t" \
111 "pmaddwd %%mm3, %%mm5 \n\t" \
112 #rounder ", %%mm0 \n\t"\
113 "paddd %%mm0, %%mm1 \n\t" \
114 "paddd %%mm0, %%mm0 \n\t" \
115 "psubd %%mm1, %%mm0 \n\t" \
116 "pmaddwd 64(%2), %%mm2 \n\t" \
117 "paddd %%mm5, %%mm7 \n\t" \
118 "movq 72(%2), %%mm5 \n\t" \
119 "pmaddwd %%mm3, %%mm5 \n\t" \
120 "paddd %%mm4, %%mm7 \n\t" \
121 "paddd %%mm4, %%mm4 \n\t" \
122 "psubd %%mm7, %%mm4 \n\t" \
123 "paddd %%mm2, %%mm5 \n\t" \
124 "psrad $" #shift ", %%mm7 \n\t"\
125 "psrad $" #shift ", %%mm4 \n\t"\
126 "movq %%mm1, %%mm2 \n\t" \
127 "paddd %%mm5, %%mm1 \n\t" \
128 "psubd %%mm5, %%mm2 \n\t" \
129 "psrad $" #shift ", %%mm1 \n\t"\
130 "psrad $" #shift ", %%mm2 \n\t"\
131 "packssdw %%mm1, %%mm7 \n\t" \
132 "packssdw %%mm4, %%mm2 \n\t" \
133 "movq %%mm7, " #dst " \n\t"\
134 "movq " #src1 ", %%mm1 \n\t" \
135 "movq 80(%2), %%mm4 \n\t" \
136 "movq %%mm2, 24+" #dst " \n\t"\
137 "pmaddwd %%mm1, %%mm4 \n\t" \
138 "movq 88(%2), %%mm7 \n\t" \
139 "pmaddwd 96(%2), %%mm1 \n\t" \
140 "pmaddwd %%mm3, %%mm7 \n\t" \
141 "movq %%mm0, %%mm2 \n\t" \
142 "pmaddwd 104(%2), %%mm3 \n\t" \
143 "paddd %%mm7, %%mm4 \n\t" \
144 "paddd %%mm4, %%mm2 \n\t" \
145 "psubd %%mm4, %%mm0 \n\t" \
146 "psrad $" #shift ", %%mm2 \n\t"\
147 "psrad $" #shift ", %%mm0 \n\t"\
148 "movq %%mm6, %%mm4 \n\t" \
149 "paddd %%mm1, %%mm3 \n\t" \
150 "paddd %%mm3, %%mm6 \n\t" \
151 "psubd %%mm3, %%mm4 \n\t" \
152 "psrad $" #shift ", %%mm6 \n\t"\
153 "packssdw %%mm6, %%mm2 \n\t" \
154 "movq %%mm2, 8+" #dst " \n\t"\
155 "psrad $" #shift ", %%mm4 \n\t"\
156 "packssdw %%mm0, %%mm4 \n\t" \
157 "movq %%mm4, 16+" #dst " \n\t"\
159 #define COL_IDCT(src0, src4, src1, src5, dst, shift) \
160 "movq " #src0 ", %%mm0 \n\t" \
161 "movq " #src4 ", %%mm1 \n\t" \
162 "movq " #src1 ", %%mm2 \n\t" \
163 "movq " #src5 ", %%mm3 \n\t" \
164 "movq 16(%2), %%mm4 \n\t" \
165 "pmaddwd %%mm0, %%mm4 \n\t" \
166 "movq 24(%2), %%mm5 \n\t" \
167 "pmaddwd %%mm5, %%mm0 \n\t" \
168 "movq 32(%2), %%mm5 \n\t" \
169 "pmaddwd %%mm1, %%mm5 \n\t" \
170 "movq 40(%2), %%mm6 \n\t" \
171 "pmaddwd %%mm6, %%mm1 \n\t" \
172 "movq %%mm4, %%mm6 \n\t" \
173 "movq 48(%2), %%mm7 \n\t" \
174 "pmaddwd %%mm2, %%mm7 \n\t" \
175 "paddd %%mm5, %%mm4 \n\t" \
176 "psubd %%mm5, %%mm6 \n\t" \
177 "movq %%mm0, %%mm5 \n\t" \
178 "paddd %%mm1, %%mm0 \n\t" \
179 "psubd %%mm1, %%mm5 \n\t" \
180 "movq 56(%2), %%mm1 \n\t" \
181 "pmaddwd %%mm3, %%mm1 \n\t" \
182 "pmaddwd 64(%2), %%mm2 \n\t" \
183 "paddd %%mm1, %%mm7 \n\t" \
184 "movq 72(%2), %%mm1 \n\t" \
185 "pmaddwd %%mm3, %%mm1 \n\t" \
186 "paddd %%mm4, %%mm7 \n\t" \
187 "paddd %%mm4, %%mm4 \n\t" \
188 "psubd %%mm7, %%mm4 \n\t" \
189 "paddd %%mm2, %%mm1 \n\t" \
190 "psrad $" #shift ", %%mm7 \n\t"\
191 "psrad $" #shift ", %%mm4 \n\t"\
192 "movq %%mm0, %%mm2 \n\t" \
193 "paddd %%mm1, %%mm0 \n\t" \
194 "psubd %%mm1, %%mm2 \n\t" \
195 "psrad $" #shift ", %%mm0 \n\t"\
196 "psrad $" #shift ", %%mm2 \n\t"\
197 "packssdw %%mm7, %%mm7 \n\t" \
198 "movd %%mm7, " #dst " \n\t"\
199 "packssdw %%mm0, %%mm0 \n\t" \
200 "movd %%mm0, 16+" #dst " \n\t"\
201 "packssdw %%mm2, %%mm2 \n\t" \
202 "movd %%mm2, 96+" #dst " \n\t"\
203 "packssdw %%mm4, %%mm4 \n\t" \
204 "movd %%mm4, 112+" #dst " \n\t"\
205 "movq " #src1 ", %%mm0 \n\t" \
206 "movq 80(%2), %%mm4 \n\t" \
207 "pmaddwd %%mm0, %%mm4 \n\t" \
208 "movq 88(%2), %%mm7 \n\t" \
209 "pmaddwd 96(%2), %%mm0 \n\t" \
210 "pmaddwd %%mm3, %%mm7 \n\t" \
211 "movq %%mm5, %%mm2 \n\t" \
212 "pmaddwd 104(%2), %%mm3 \n\t" \
213 "paddd %%mm7, %%mm4 \n\t" \
214 "paddd %%mm4, %%mm2 \n\t" \
215 "psubd %%mm4, %%mm5 \n\t" \
216 "psrad $" #shift ", %%mm2 \n\t"\
217 "psrad $" #shift ", %%mm5 \n\t"\
218 "movq %%mm6, %%mm4 \n\t" \
219 "paddd %%mm0, %%mm3 \n\t" \
220 "paddd %%mm3, %%mm6 \n\t" \
221 "psubd %%mm3, %%mm4 \n\t" \
222 "psrad $" #shift ", %%mm6 \n\t"\
223 "psrad $" #shift ", %%mm4 \n\t"\
224 "packssdw %%mm2, %%mm2 \n\t" \
225 "packssdw %%mm6, %%mm6 \n\t" \
226 "movd %%mm2, 32+" #dst " \n\t"\
227 "packssdw %%mm4, %%mm4 \n\t" \
228 "packssdw %%mm5, %%mm5 \n\t" \
229 "movd %%mm6, 48+" #dst " \n\t"\
230 "movd %%mm4, 64+" #dst " \n\t"\
231 "movd %%mm5, 80+" #dst " \n\t"\
234 #define DC_COND_ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
235 "movq " #src0 ", %%mm0 \n\t" \
236 "movq " #src4 ", %%mm1 \n\t" \
237 "movq " #src1 ", %%mm2 \n\t" \
238 "movq " #src5 ", %%mm3 \n\t" \
239 "movq "MANGLE(wm1010)", %%mm4 \n\t"\
240 "pand %%mm0, %%mm4 \n\t"\
241 "por %%mm1, %%mm4 \n\t"\
242 "por %%mm2, %%mm4 \n\t"\
243 "por %%mm3, %%mm4 \n\t"\
244 "packssdw %%mm4,%%mm4 \n\t"\
245 "movd %%mm4, %%eax \n\t"\
246 "orl %%eax, %%eax \n\t"\
248 "movq 16(%2), %%mm4 \n\t" \
249 "pmaddwd %%mm0, %%mm4 \n\t" \
250 "movq 24(%2), %%mm5 \n\t" \
251 "pmaddwd %%mm5, %%mm0 \n\t" \
252 "movq 32(%2), %%mm5 \n\t" \
253 "pmaddwd %%mm1, %%mm5 \n\t" \
254 "movq 40(%2), %%mm6 \n\t" \
255 "pmaddwd %%mm6, %%mm1 \n\t" \
256 "movq 48(%2), %%mm7 \n\t" \
257 "pmaddwd %%mm2, %%mm7 \n\t" \
258 #rounder ", %%mm4 \n\t"\
259 "movq %%mm4, %%mm6 \n\t" \
260 "paddd %%mm5, %%mm4 \n\t" \
261 "psubd %%mm5, %%mm6 \n\t" \
262 "movq 56(%2), %%mm5 \n\t" \
263 "pmaddwd %%mm3, %%mm5 \n\t" \
264 #rounder ", %%mm0 \n\t"\
265 "paddd %%mm0, %%mm1 \n\t" \
266 "paddd %%mm0, %%mm0 \n\t" \
267 "psubd %%mm1, %%mm0 \n\t" \
268 "pmaddwd 64(%2), %%mm2 \n\t" \
269 "paddd %%mm5, %%mm7 \n\t" \
270 "movq 72(%2), %%mm5 \n\t" \
271 "pmaddwd %%mm3, %%mm5 \n\t" \
272 "paddd %%mm4, %%mm7 \n\t" \
273 "paddd %%mm4, %%mm4 \n\t" \
274 "psubd %%mm7, %%mm4 \n\t" \
275 "paddd %%mm2, %%mm5 \n\t" \
276 "psrad $" #shift ", %%mm7 \n\t"\
277 "psrad $" #shift ", %%mm4 \n\t"\
278 "movq %%mm1, %%mm2 \n\t" \
279 "paddd %%mm5, %%mm1 \n\t" \
280 "psubd %%mm5, %%mm2 \n\t" \
281 "psrad $" #shift ", %%mm1 \n\t"\
282 "psrad $" #shift ", %%mm2 \n\t"\
283 "packssdw %%mm1, %%mm7 \n\t" \
284 "packssdw %%mm4, %%mm2 \n\t" \
285 "movq %%mm7, " #dst " \n\t"\
286 "movq " #src1 ", %%mm1 \n\t" \
287 "movq 80(%2), %%mm4 \n\t" \
288 "movq %%mm2, 24+" #dst " \n\t"\
289 "pmaddwd %%mm1, %%mm4 \n\t" \
290 "movq 88(%2), %%mm7 \n\t" \
291 "pmaddwd 96(%2), %%mm1 \n\t" \
292 "pmaddwd %%mm3, %%mm7 \n\t" \
293 "movq %%mm0, %%mm2 \n\t" \
294 "pmaddwd 104(%2), %%mm3 \n\t" \
295 "paddd %%mm7, %%mm4 \n\t" \
296 "paddd %%mm4, %%mm2 \n\t" \
297 "psubd %%mm4, %%mm0 \n\t" \
298 "psrad $" #shift ", %%mm2 \n\t"\
299 "psrad $" #shift ", %%mm0 \n\t"\
300 "movq %%mm6, %%mm4 \n\t" \
301 "paddd %%mm1, %%mm3 \n\t" \
302 "paddd %%mm3, %%mm6 \n\t" \
303 "psubd %%mm3, %%mm4 \n\t" \
304 "psrad $" #shift ", %%mm6 \n\t"\
305 "packssdw %%mm6, %%mm2 \n\t" \
306 "movq %%mm2, 8+" #dst " \n\t"\
307 "psrad $" #shift ", %%mm4 \n\t"\
308 "packssdw %%mm0, %%mm4 \n\t" \
309 "movq %%mm4, 16+" #dst " \n\t"\
312 "pslld $16, %%mm0 \n\t"\
313 "#paddd "MANGLE(d40000)", %%mm0 \n\t"\
314 "psrad $13, %%mm0 \n\t"\
315 "packssdw %%mm0, %%mm0 \n\t"\
316 "movq %%mm0, " #dst " \n\t"\
317 "movq %%mm0, 8+" #dst " \n\t"\
318 "movq %%mm0, 16+" #dst " \n\t"\
319 "movq %%mm0, 24+" #dst " \n\t"\
324 ROW_IDCT( (%0), 8(%0), 16(%0), 24(%0), 0(%1),paddd 8(%2), 11)
329 DC_COND_ROW_IDCT( 32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11)
330 DC_COND_ROW_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11)
331 DC_COND_ROW_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11)
335 COL_IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
336 COL_IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
337 COL_IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
338 COL_IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
342 #define DC_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
343 "movq " #src0 ", %%mm0 \n\t" \
344 "movq " #src4 ", %%mm1 \n\t" \
345 "movq " #src1 ", %%mm2 \n\t" \
346 "movq " #src5 ", %%mm3 \n\t" \
347 "movq "MANGLE(wm1010)", %%mm4 \n\t"\
348 "pand %%mm0, %%mm4 \n\t"\
349 "por %%mm1, %%mm4 \n\t"\
350 "por %%mm2, %%mm4 \n\t"\
351 "por %%mm3, %%mm4 \n\t"\
352 "packssdw %%mm4,%%mm4 \n\t"\
353 "movd %%mm4, %%eax \n\t"\
354 "orl %%eax, %%eax \n\t"\
356 "movq 16(%2), %%mm4 \n\t" \
357 "pmaddwd %%mm0, %%mm4 \n\t" \
358 "movq 24(%2), %%mm5 \n\t" \
359 "pmaddwd %%mm5, %%mm0 \n\t" \
360 "movq 32(%2), %%mm5 \n\t" \
361 "pmaddwd %%mm1, %%mm5 \n\t" \
362 "movq 40(%2), %%mm6 \n\t" \
363 "pmaddwd %%mm6, %%mm1 \n\t" \
364 "movq 48(%2), %%mm7 \n\t" \
365 "pmaddwd %%mm2, %%mm7 \n\t" \
366 #rounder ", %%mm4 \n\t"\
367 "movq %%mm4, %%mm6 \n\t" \
368 "paddd %%mm5, %%mm4 \n\t" \
369 "psubd %%mm5, %%mm6 \n\t" \
370 "movq 56(%2), %%mm5 \n\t" \
371 "pmaddwd %%mm3, %%mm5 \n\t" \
372 #rounder ", %%mm0 \n\t"\
373 "paddd %%mm0, %%mm1 \n\t" \
374 "paddd %%mm0, %%mm0 \n\t" \
375 "psubd %%mm1, %%mm0 \n\t" \
376 "pmaddwd 64(%2), %%mm2 \n\t" \
377 "paddd %%mm5, %%mm7 \n\t" \
378 "movq 72(%2), %%mm5 \n\t" \
379 "pmaddwd %%mm3, %%mm5 \n\t" \
380 "paddd %%mm4, %%mm7 \n\t" \
381 "paddd %%mm4, %%mm4 \n\t" \
382 "psubd %%mm7, %%mm4 \n\t" \
383 "paddd %%mm2, %%mm5 \n\t" \
384 "psrad $" #shift ", %%mm7 \n\t"\
385 "psrad $" #shift ", %%mm4 \n\t"\
386 "movq %%mm1, %%mm2 \n\t" \
387 "paddd %%mm5, %%mm1 \n\t" \
388 "psubd %%mm5, %%mm2 \n\t" \
389 "psrad $" #shift ", %%mm1 \n\t"\
390 "psrad $" #shift ", %%mm2 \n\t"\
391 "packssdw %%mm1, %%mm7 \n\t" \
392 "packssdw %%mm4, %%mm2 \n\t" \
393 "movq %%mm7, " #dst " \n\t"\
394 "movq " #src1 ", %%mm1 \n\t" \
395 "movq 80(%2), %%mm4 \n\t" \
396 "movq %%mm2, 24+" #dst " \n\t"\
397 "pmaddwd %%mm1, %%mm4 \n\t" \
398 "movq 88(%2), %%mm7 \n\t" \
399 "pmaddwd 96(%2), %%mm1 \n\t" \
400 "pmaddwd %%mm3, %%mm7 \n\t" \
401 "movq %%mm0, %%mm2 \n\t" \
402 "pmaddwd 104(%2), %%mm3 \n\t" \
403 "paddd %%mm7, %%mm4 \n\t" \
404 "paddd %%mm4, %%mm2 \n\t" \
405 "psubd %%mm4, %%mm0 \n\t" \
406 "psrad $" #shift ", %%mm2 \n\t"\
407 "psrad $" #shift ", %%mm0 \n\t"\
408 "movq %%mm6, %%mm4 \n\t" \
409 "paddd %%mm1, %%mm3 \n\t" \
410 "paddd %%mm3, %%mm6 \n\t" \
411 "psubd %%mm3, %%mm4 \n\t" \
412 "psrad $" #shift ", %%mm6 \n\t"\
413 "packssdw %%mm6, %%mm2 \n\t" \
414 "movq %%mm2, 8+" #dst " \n\t"\
415 "psrad $" #shift ", %%mm4 \n\t"\
416 "packssdw %%mm0, %%mm4 \n\t" \
417 "movq %%mm4, 16+" #dst " \n\t"\
420 "pslld $16, %%mm0 \n\t"\
421 "paddd "MANGLE(d40000)", %%mm0 \n\t"\
422 "psrad $13, %%mm0 \n\t"\
423 "packssdw %%mm0, %%mm0 \n\t"\
424 "movq %%mm0, " #dst " \n\t"\
425 "movq %%mm0, 8+" #dst " \n\t"\
426 "movq %%mm0, 16+" #dst " \n\t"\
427 "movq %%mm0, 24+" #dst " \n\t"\
430 #define Z_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift, bt) \
431 "movq " #src0 ", %%mm0 \n\t" \
432 "movq " #src4 ", %%mm1 \n\t" \
433 "movq " #src1 ", %%mm2 \n\t" \
434 "movq " #src5 ", %%mm3 \n\t" \
435 "movq %%mm0, %%mm4 \n\t"\
436 "por %%mm1, %%mm4 \n\t"\
437 "por %%mm2, %%mm4 \n\t"\
438 "por %%mm3, %%mm4 \n\t"\
439 "packssdw %%mm4,%%mm4 \n\t"\
440 "movd %%mm4, %%eax \n\t"\
441 "orl %%eax, %%eax \n\t"\
443 "movq 16(%2), %%mm4 \n\t" \
444 "pmaddwd %%mm0, %%mm4 \n\t" \
445 "movq 24(%2), %%mm5 \n\t" \
446 "pmaddwd %%mm5, %%mm0 \n\t" \
447 "movq 32(%2), %%mm5 \n\t" \
448 "pmaddwd %%mm1, %%mm5 \n\t" \
449 "movq 40(%2), %%mm6 \n\t" \
450 "pmaddwd %%mm6, %%mm1 \n\t" \
451 "movq 48(%2), %%mm7 \n\t" \
452 "pmaddwd %%mm2, %%mm7 \n\t" \
453 #rounder ", %%mm4 \n\t"\
454 "movq %%mm4, %%mm6 \n\t" \
455 "paddd %%mm5, %%mm4 \n\t" \
456 "psubd %%mm5, %%mm6 \n\t" \
457 "movq 56(%2), %%mm5 \n\t" \
458 "pmaddwd %%mm3, %%mm5 \n\t" \
459 #rounder ", %%mm0 \n\t"\
460 "paddd %%mm0, %%mm1 \n\t" \
461 "paddd %%mm0, %%mm0 \n\t" \
462 "psubd %%mm1, %%mm0 \n\t" \
463 "pmaddwd 64(%2), %%mm2 \n\t" \
464 "paddd %%mm5, %%mm7 \n\t" \
465 "movq 72(%2), %%mm5 \n\t" \
466 "pmaddwd %%mm3, %%mm5 \n\t" \
467 "paddd %%mm4, %%mm7 \n\t" \
468 "paddd %%mm4, %%mm4 \n\t" \
469 "psubd %%mm7, %%mm4 \n\t" \
470 "paddd %%mm2, %%mm5 \n\t" \
471 "psrad $" #shift ", %%mm7 \n\t"\
472 "psrad $" #shift ", %%mm4 \n\t"\
473 "movq %%mm1, %%mm2 \n\t" \
474 "paddd %%mm5, %%mm1 \n\t" \
475 "psubd %%mm5, %%mm2 \n\t" \
476 "psrad $" #shift ", %%mm1 \n\t"\
477 "psrad $" #shift ", %%mm2 \n\t"\
478 "packssdw %%mm1, %%mm7 \n\t" \
479 "packssdw %%mm4, %%mm2 \n\t" \
480 "movq %%mm7, " #dst " \n\t"\
481 "movq " #src1 ", %%mm1 \n\t" \
482 "movq 80(%2), %%mm4 \n\t" \
483 "movq %%mm2, 24+" #dst " \n\t"\
484 "pmaddwd %%mm1, %%mm4 \n\t" \
485 "movq 88(%2), %%mm7 \n\t" \
486 "pmaddwd 96(%2), %%mm1 \n\t" \
487 "pmaddwd %%mm3, %%mm7 \n\t" \
488 "movq %%mm0, %%mm2 \n\t" \
489 "pmaddwd 104(%2), %%mm3 \n\t" \
490 "paddd %%mm7, %%mm4 \n\t" \
491 "paddd %%mm4, %%mm2 \n\t" \
492 "psubd %%mm4, %%mm0 \n\t" \
493 "psrad $" #shift ", %%mm2 \n\t"\
494 "psrad $" #shift ", %%mm0 \n\t"\
495 "movq %%mm6, %%mm4 \n\t" \
496 "paddd %%mm1, %%mm3 \n\t" \
497 "paddd %%mm3, %%mm6 \n\t" \
498 "psubd %%mm3, %%mm4 \n\t" \
499 "psrad $" #shift ", %%mm6 \n\t"\
500 "packssdw %%mm6, %%mm2 \n\t" \
501 "movq %%mm2, 8+" #dst " \n\t"\
502 "psrad $" #shift ", %%mm4 \n\t"\
503 "packssdw %%mm0, %%mm4 \n\t" \
504 "movq %%mm4, 16+" #dst " \n\t"\
506 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
507 "movq " #src0 ", %%mm0 \n\t" \
508 "movq " #src4 ", %%mm1 \n\t" \
509 "movq " #src1 ", %%mm2 \n\t" \
510 "movq " #src5 ", %%mm3 \n\t" \
511 "movq 16(%2), %%mm4 \n\t" \
512 "pmaddwd %%mm0, %%mm4 \n\t" \
513 "movq 24(%2), %%mm5 \n\t" \
514 "pmaddwd %%mm5, %%mm0 \n\t" \
515 "movq 32(%2), %%mm5 \n\t" \
516 "pmaddwd %%mm1, %%mm5 \n\t" \
517 "movq 40(%2), %%mm6 \n\t" \
518 "pmaddwd %%mm6, %%mm1 \n\t" \
519 "movq 48(%2), %%mm7 \n\t" \
520 "pmaddwd %%mm2, %%mm7 \n\t" \
521 #rounder ", %%mm4 \n\t"\
522 "movq %%mm4, %%mm6 \n\t" \
523 "paddd %%mm5, %%mm4 \n\t" \
524 "psubd %%mm5, %%mm6 \n\t" \
525 "movq 56(%2), %%mm5 \n\t" \
526 "pmaddwd %%mm3, %%mm5 \n\t" \
527 #rounder ", %%mm0 \n\t"\
528 "paddd %%mm0, %%mm1 \n\t" \
529 "paddd %%mm0, %%mm0 \n\t" \
530 "psubd %%mm1, %%mm0 \n\t" \
531 "pmaddwd 64(%2), %%mm2 \n\t" \
532 "paddd %%mm5, %%mm7 \n\t" \
533 "movq 72(%2), %%mm5 \n\t" \
534 "pmaddwd %%mm3, %%mm5 \n\t" \
535 "paddd %%mm4, %%mm7 \n\t" \
536 "paddd %%mm4, %%mm4 \n\t" \
537 "psubd %%mm7, %%mm4 \n\t" \
538 "paddd %%mm2, %%mm5 \n\t" \
539 "psrad $" #shift ", %%mm7 \n\t"\
540 "psrad $" #shift ", %%mm4 \n\t"\
541 "movq %%mm1, %%mm2 \n\t" \
542 "paddd %%mm5, %%mm1 \n\t" \
543 "psubd %%mm5, %%mm2 \n\t" \
544 "psrad $" #shift ", %%mm1 \n\t"\
545 "psrad $" #shift ", %%mm2 \n\t"\
546 "packssdw %%mm1, %%mm7 \n\t" \
547 "packssdw %%mm4, %%mm2 \n\t" \
548 "movq %%mm7, " #dst " \n\t"\
549 "movq " #src1 ", %%mm1 \n\t" \
550 "movq 80(%2), %%mm4 \n\t" \
551 "movq %%mm2, 24+" #dst " \n\t"\
552 "pmaddwd %%mm1, %%mm4 \n\t" \
553 "movq 88(%2), %%mm7 \n\t" \
554 "pmaddwd 96(%2), %%mm1 \n\t" \
555 "pmaddwd %%mm3, %%mm7 \n\t" \
556 "movq %%mm0, %%mm2 \n\t" \
557 "pmaddwd 104(%2), %%mm3 \n\t" \
558 "paddd %%mm7, %%mm4 \n\t" \
559 "paddd %%mm4, %%mm2 \n\t" \
560 "psubd %%mm4, %%mm0 \n\t" \
561 "psrad $" #shift ", %%mm2 \n\t"\
562 "psrad $" #shift ", %%mm0 \n\t"\
563 "movq %%mm6, %%mm4 \n\t" \
564 "paddd %%mm1, %%mm3 \n\t" \
565 "paddd %%mm3, %%mm6 \n\t" \
566 "psubd %%mm3, %%mm4 \n\t" \
567 "psrad $" #shift ", %%mm6 \n\t"\
568 "packssdw %%mm6, %%mm2 \n\t" \
569 "movq %%mm2, 8+" #dst " \n\t"\
570 "psrad $" #shift ", %%mm4 \n\t"\
571 "packssdw %%mm0, %%mm4 \n\t" \
572 "movq %%mm4, 16+" #dst " \n\t"\
575 DC_COND_IDCT( 0(%0), 8(%0), 16(%0), 24(%0), 0(%1),paddd 8(%2), 11)
576 Z_COND_IDCT( 32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11, 4f)
577 Z_COND_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 2f)
578 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 1f)
581 #define IDCT(src0, src4, src1, src5, dst, shift) \
582 "movq " #src0 ", %%mm0 \n\t" \
583 "movq " #src4 ", %%mm1 \n\t" \
584 "movq " #src1 ", %%mm2 \n\t" \
585 "movq " #src5 ", %%mm3 \n\t" \
586 "movq 16(%2), %%mm4 \n\t" \
587 "pmaddwd %%mm0, %%mm4 \n\t" \
588 "movq 24(%2), %%mm5 \n\t" \
589 "pmaddwd %%mm5, %%mm0 \n\t" \
590 "movq 32(%2), %%mm5 \n\t" \
591 "pmaddwd %%mm1, %%mm5 \n\t" \
592 "movq 40(%2), %%mm6 \n\t" \
593 "pmaddwd %%mm6, %%mm1 \n\t" \
594 "movq %%mm4, %%mm6 \n\t" \
595 "movq 48(%2), %%mm7 \n\t" \
596 "pmaddwd %%mm2, %%mm7 \n\t" \
597 "paddd %%mm5, %%mm4 \n\t" \
598 "psubd %%mm5, %%mm6 \n\t" \
599 "movq %%mm0, %%mm5 \n\t" \
600 "paddd %%mm1, %%mm0 \n\t" \
601 "psubd %%mm1, %%mm5 \n\t" \
602 "movq 56(%2), %%mm1 \n\t" \
603 "pmaddwd %%mm3, %%mm1 \n\t" \
604 "pmaddwd 64(%2), %%mm2 \n\t" \
605 "paddd %%mm1, %%mm7 \n\t" \
606 "movq 72(%2), %%mm1 \n\t" \
607 "pmaddwd %%mm3, %%mm1 \n\t" \
608 "paddd %%mm4, %%mm7 \n\t" \
609 "paddd %%mm4, %%mm4 \n\t" \
610 "psubd %%mm7, %%mm4 \n\t" \
611 "paddd %%mm2, %%mm1 \n\t" \
612 "psrad $" #shift ", %%mm7 \n\t"\
613 "psrad $" #shift ", %%mm4 \n\t"\
614 "movq %%mm0, %%mm2 \n\t" \
615 "paddd %%mm1, %%mm0 \n\t" \
616 "psubd %%mm1, %%mm2 \n\t" \
617 "psrad $" #shift ", %%mm0 \n\t"\
618 "psrad $" #shift ", %%mm2 \n\t"\
619 "packssdw %%mm7, %%mm7 \n\t" \
620 "movd %%mm7, " #dst " \n\t"\
621 "packssdw %%mm0, %%mm0 \n\t" \
622 "movd %%mm0, 16+" #dst " \n\t"\
623 "packssdw %%mm2, %%mm2 \n\t" \
624 "movd %%mm2, 96+" #dst " \n\t"\
625 "packssdw %%mm4, %%mm4 \n\t" \
626 "movd %%mm4, 112+" #dst " \n\t"\
627 "movq " #src1 ", %%mm0 \n\t" \
628 "movq 80(%2), %%mm4 \n\t" \
629 "pmaddwd %%mm0, %%mm4 \n\t" \
630 "movq 88(%2), %%mm7 \n\t" \
631 "pmaddwd 96(%2), %%mm0 \n\t" \
632 "pmaddwd %%mm3, %%mm7 \n\t" \
633 "movq %%mm5, %%mm2 \n\t" \
634 "pmaddwd 104(%2), %%mm3 \n\t" \
635 "paddd %%mm7, %%mm4 \n\t" \
636 "paddd %%mm4, %%mm2 \n\t" \
637 "psubd %%mm4, %%mm5 \n\t" \
638 "psrad $" #shift ", %%mm2 \n\t"\
639 "psrad $" #shift ", %%mm5 \n\t"\
640 "movq %%mm6, %%mm4 \n\t" \
641 "paddd %%mm0, %%mm3 \n\t" \
642 "paddd %%mm3, %%mm6 \n\t" \
643 "psubd %%mm3, %%mm4 \n\t" \
644 "psrad $" #shift ", %%mm6 \n\t"\
645 "psrad $" #shift ", %%mm4 \n\t"\
646 "packssdw %%mm2, %%mm2 \n\t" \
647 "packssdw %%mm6, %%mm6 \n\t" \
648 "movd %%mm2, 32+" #dst " \n\t"\
649 "packssdw %%mm4, %%mm4 \n\t" \
650 "packssdw %%mm5, %%mm5 \n\t" \
651 "movd %%mm6, 48+" #dst " \n\t"\
652 "movd %%mm4, 64+" #dst " \n\t"\
653 "movd %%mm5, 80+" #dst " \n\t"
657 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
658 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
659 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
660 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
665 Z_COND_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 6f)
666 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 5f)
669 #define IDCT(src0, src4, src1, src5, dst, shift) \
670 "movq " #src0 ", %%mm0 \n\t" \
671 "movq " #src4 ", %%mm1 \n\t" \
672 "movq " #src5 ", %%mm3 \n\t" \
673 "movq 16(%2), %%mm4 \n\t" \
674 "pmaddwd %%mm0, %%mm4 \n\t" \
675 "movq 24(%2), %%mm5 \n\t" \
676 "pmaddwd %%mm5, %%mm0 \n\t" \
677 "movq 32(%2), %%mm5 \n\t" \
678 "pmaddwd %%mm1, %%mm5 \n\t" \
679 "movq 40(%2), %%mm6 \n\t" \
680 "pmaddwd %%mm6, %%mm1 \n\t" \
681 "movq %%mm4, %%mm6 \n\t" \
682 "paddd %%mm5, %%mm4 \n\t" \
683 "psubd %%mm5, %%mm6 \n\t" \
684 "movq %%mm0, %%mm5 \n\t" \
685 "paddd %%mm1, %%mm0 \n\t" \
686 "psubd %%mm1, %%mm5 \n\t" \
687 "movq 56(%2), %%mm1 \n\t" \
688 "pmaddwd %%mm3, %%mm1 \n\t" \
689 "movq 72(%2), %%mm7 \n\t" \
690 "pmaddwd %%mm3, %%mm7 \n\t" \
691 "paddd %%mm4, %%mm1 \n\t" \
692 "paddd %%mm4, %%mm4 \n\t" \
693 "psubd %%mm1, %%mm4 \n\t" \
694 "psrad $" #shift ", %%mm1 \n\t"\
695 "psrad $" #shift ", %%mm4 \n\t"\
696 "movq %%mm0, %%mm2 \n\t" \
697 "paddd %%mm7, %%mm0 \n\t" \
698 "psubd %%mm7, %%mm2 \n\t" \
699 "psrad $" #shift ", %%mm0 \n\t"\
700 "psrad $" #shift ", %%mm2 \n\t"\
701 "packssdw %%mm1, %%mm1 \n\t" \
702 "movd %%mm1, " #dst " \n\t"\
703 "packssdw %%mm0, %%mm0 \n\t" \
704 "movd %%mm0, 16+" #dst " \n\t"\
705 "packssdw %%mm2, %%mm2 \n\t" \
706 "movd %%mm2, 96+" #dst " \n\t"\
707 "packssdw %%mm4, %%mm4 \n\t" \
708 "movd %%mm4, 112+" #dst " \n\t"\
709 "movq 88(%2), %%mm1 \n\t" \
710 "pmaddwd %%mm3, %%mm1 \n\t" \
711 "movq %%mm5, %%mm2 \n\t" \
712 "pmaddwd 104(%2), %%mm3 \n\t" \
713 "paddd %%mm1, %%mm2 \n\t" \
714 "psubd %%mm1, %%mm5 \n\t" \
715 "psrad $" #shift ", %%mm2 \n\t"\
716 "psrad $" #shift ", %%mm5 \n\t"\
717 "movq %%mm6, %%mm1 \n\t" \
718 "paddd %%mm3, %%mm6 \n\t" \
719 "psubd %%mm3, %%mm1 \n\t" \
720 "psrad $" #shift ", %%mm6 \n\t"\
721 "psrad $" #shift ", %%mm1 \n\t"\
722 "packssdw %%mm2, %%mm2 \n\t" \
723 "packssdw %%mm6, %%mm6 \n\t" \
724 "movd %%mm2, 32+" #dst " \n\t"\
725 "packssdw %%mm1, %%mm1 \n\t" \
726 "packssdw %%mm5, %%mm5 \n\t" \
727 "movd %%mm6, 48+" #dst " \n\t"\
728 "movd %%mm1, 64+" #dst " \n\t"\
729 "movd %%mm5, 80+" #dst " \n\t"
732 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
733 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
734 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
735 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
740 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 7f)
743 #define IDCT(src0, src4, src1, src5, dst, shift) \
744 "movq " #src0 ", %%mm0 \n\t" \
745 "movq " #src5 ", %%mm3 \n\t" \
746 "movq 16(%2), %%mm4 \n\t" \
747 "pmaddwd %%mm0, %%mm4 \n\t" \
748 "movq 24(%2), %%mm5 \n\t" \
749 "pmaddwd %%mm5, %%mm0 \n\t" \
750 "movq %%mm4, %%mm6 \n\t" \
751 "movq %%mm0, %%mm5 \n\t" \
752 "movq 56(%2), %%mm1 \n\t" \
753 "pmaddwd %%mm3, %%mm1 \n\t" \
754 "movq 72(%2), %%mm7 \n\t" \
755 "pmaddwd %%mm3, %%mm7 \n\t" \
756 "paddd %%mm4, %%mm1 \n\t" \
757 "paddd %%mm4, %%mm4 \n\t" \
758 "psubd %%mm1, %%mm4 \n\t" \
759 "psrad $" #shift ", %%mm1 \n\t"\
760 "psrad $" #shift ", %%mm4 \n\t"\
761 "movq %%mm0, %%mm2 \n\t" \
762 "paddd %%mm7, %%mm0 \n\t" \
763 "psubd %%mm7, %%mm2 \n\t" \
764 "psrad $" #shift ", %%mm0 \n\t"\
765 "psrad $" #shift ", %%mm2 \n\t"\
766 "packssdw %%mm1, %%mm1 \n\t" \
767 "movd %%mm1, " #dst " \n\t"\
768 "packssdw %%mm0, %%mm0 \n\t" \
769 "movd %%mm0, 16+" #dst " \n\t"\
770 "packssdw %%mm2, %%mm2 \n\t" \
771 "movd %%mm2, 96+" #dst " \n\t"\
772 "packssdw %%mm4, %%mm4 \n\t" \
773 "movd %%mm4, 112+" #dst " \n\t"\
774 "movq 88(%2), %%mm1 \n\t" \
775 "pmaddwd %%mm3, %%mm1 \n\t" \
776 "movq %%mm5, %%mm2 \n\t" \
777 "pmaddwd 104(%2), %%mm3 \n\t" \
778 "paddd %%mm1, %%mm2 \n\t" \
779 "psubd %%mm1, %%mm5 \n\t" \
780 "psrad $" #shift ", %%mm2 \n\t"\
781 "psrad $" #shift ", %%mm5 \n\t"\
782 "movq %%mm6, %%mm1 \n\t" \
783 "paddd %%mm3, %%mm6 \n\t" \
784 "psubd %%mm3, %%mm1 \n\t" \
785 "psrad $" #shift ", %%mm6 \n\t"\
786 "psrad $" #shift ", %%mm1 \n\t"\
787 "packssdw %%mm2, %%mm2 \n\t" \
788 "packssdw %%mm6, %%mm6 \n\t" \
789 "movd %%mm2, 32+" #dst " \n\t"\
790 "packssdw %%mm1, %%mm1 \n\t" \
791 "packssdw %%mm5, %%mm5 \n\t" \
792 "movd %%mm6, 48+" #dst " \n\t"\
793 "movd %%mm1, 64+" #dst " \n\t"\
794 "movd %%mm5, 80+" #dst " \n\t"
798 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
799 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
800 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
801 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
806 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 3f)
809 #define IDCT(src0, src4, src1, src5, dst, shift) \
810 "movq " #src0 ", %%mm0 \n\t" \
811 "movq " #src1 ", %%mm2 \n\t" \
812 "movq " #src5 ", %%mm3 \n\t" \
813 "movq 16(%2), %%mm4 \n\t" \
814 "pmaddwd %%mm0, %%mm4 \n\t" \
815 "movq 24(%2), %%mm5 \n\t" \
816 "pmaddwd %%mm5, %%mm0 \n\t" \
817 "movq %%mm4, %%mm6 \n\t" \
818 "movq 48(%2), %%mm7 \n\t" \
819 "pmaddwd %%mm2, %%mm7 \n\t" \
820 "movq %%mm0, %%mm5 \n\t" \
821 "movq 56(%2), %%mm1 \n\t" \
822 "pmaddwd %%mm3, %%mm1 \n\t" \
823 "pmaddwd 64(%2), %%mm2 \n\t" \
824 "paddd %%mm1, %%mm7 \n\t" \
825 "movq 72(%2), %%mm1 \n\t" \
826 "pmaddwd %%mm3, %%mm1 \n\t" \
827 "paddd %%mm4, %%mm7 \n\t" \
828 "paddd %%mm4, %%mm4 \n\t" \
829 "psubd %%mm7, %%mm4 \n\t" \
830 "paddd %%mm2, %%mm1 \n\t" \
831 "psrad $" #shift ", %%mm7 \n\t"\
832 "psrad $" #shift ", %%mm4 \n\t"\
833 "movq %%mm0, %%mm2 \n\t" \
834 "paddd %%mm1, %%mm0 \n\t" \
835 "psubd %%mm1, %%mm2 \n\t" \
836 "psrad $" #shift ", %%mm0 \n\t"\
837 "psrad $" #shift ", %%mm2 \n\t"\
838 "packssdw %%mm7, %%mm7 \n\t" \
839 "movd %%mm7, " #dst " \n\t"\
840 "packssdw %%mm0, %%mm0 \n\t" \
841 "movd %%mm0, 16+" #dst " \n\t"\
842 "packssdw %%mm2, %%mm2 \n\t" \
843 "movd %%mm2, 96+" #dst " \n\t"\
844 "packssdw %%mm4, %%mm4 \n\t" \
845 "movd %%mm4, 112+" #dst " \n\t"\
846 "movq " #src1 ", %%mm0 \n\t" \
847 "movq 80(%2), %%mm4 \n\t" \
848 "pmaddwd %%mm0, %%mm4 \n\t" \
849 "movq 88(%2), %%mm7 \n\t" \
850 "pmaddwd 96(%2), %%mm0 \n\t" \
851 "pmaddwd %%mm3, %%mm7 \n\t" \
852 "movq %%mm5, %%mm2 \n\t" \
853 "pmaddwd 104(%2), %%mm3 \n\t" \
854 "paddd %%mm7, %%mm4 \n\t" \
855 "paddd %%mm4, %%mm2 \n\t" \
856 "psubd %%mm4, %%mm5 \n\t" \
857 "psrad $" #shift ", %%mm2 \n\t"\
858 "psrad $" #shift ", %%mm5 \n\t"\
859 "movq %%mm6, %%mm4 \n\t" \
860 "paddd %%mm0, %%mm3 \n\t" \
861 "paddd %%mm3, %%mm6 \n\t" \
862 "psubd %%mm3, %%mm4 \n\t" \
863 "psrad $" #shift ", %%mm6 \n\t"\
864 "psrad $" #shift ", %%mm4 \n\t"\
865 "packssdw %%mm2, %%mm2 \n\t" \
866 "packssdw %%mm6, %%mm6 \n\t" \
867 "movd %%mm2, 32+" #dst " \n\t"\
868 "packssdw %%mm4, %%mm4 \n\t" \
869 "packssdw %%mm5, %%mm5 \n\t" \
870 "movd %%mm6, 48+" #dst " \n\t"\
871 "movd %%mm4, 64+" #dst " \n\t"\
872 "movd %%mm5, 80+" #dst " \n\t"
875 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
876 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
877 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
878 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
884 #define IDCT(src0, src4, src1, src5, dst, shift) \
885 "movq " #src0 ", %%mm0 \n\t" \
886 "movq " #src1 ", %%mm2 \n\t" \
887 "movq 16(%2), %%mm4 \n\t" \
888 "pmaddwd %%mm0, %%mm4 \n\t" \
889 "movq 24(%2), %%mm5 \n\t" \
890 "pmaddwd %%mm5, %%mm0 \n\t" \
891 "movq %%mm4, %%mm6 \n\t" \
892 "movq 48(%2), %%mm7 \n\t" \
893 "pmaddwd %%mm2, %%mm7 \n\t" \
894 "movq %%mm0, %%mm5 \n\t" \
895 "movq 64(%2), %%mm3 \n\t"\
896 "pmaddwd %%mm2, %%mm3 \n\t" \
897 "paddd %%mm4, %%mm7 \n\t" \
898 "paddd %%mm4, %%mm4 \n\t" \
899 "psubd %%mm7, %%mm4 \n\t" \
900 "psrad $" #shift ", %%mm7 \n\t"\
901 "psrad $" #shift ", %%mm4 \n\t"\
902 "movq %%mm0, %%mm1 \n\t" \
903 "paddd %%mm3, %%mm0 \n\t" \
904 "psubd %%mm3, %%mm1 \n\t" \
905 "psrad $" #shift ", %%mm0 \n\t"\
906 "psrad $" #shift ", %%mm1 \n\t"\
907 "packssdw %%mm7, %%mm7 \n\t" \
908 "movd %%mm7, " #dst " \n\t"\
909 "packssdw %%mm0, %%mm0 \n\t" \
910 "movd %%mm0, 16+" #dst " \n\t"\
911 "packssdw %%mm1, %%mm1 \n\t" \
912 "movd %%mm1, 96+" #dst " \n\t"\
913 "packssdw %%mm4, %%mm4 \n\t" \
914 "movd %%mm4, 112+" #dst " \n\t"\
915 "movq 80(%2), %%mm4 \n\t" \
916 "pmaddwd %%mm2, %%mm4 \n\t" \
917 "pmaddwd 96(%2), %%mm2 \n\t" \
918 "movq %%mm5, %%mm1 \n\t" \
919 "paddd %%mm4, %%mm1 \n\t" \
920 "psubd %%mm4, %%mm5 \n\t" \
921 "psrad $" #shift ", %%mm1 \n\t"\
922 "psrad $" #shift ", %%mm5 \n\t"\
923 "movq %%mm6, %%mm4 \n\t" \
924 "paddd %%mm2, %%mm6 \n\t" \
925 "psubd %%mm2, %%mm4 \n\t" \
926 "psrad $" #shift ", %%mm6 \n\t"\
927 "psrad $" #shift ", %%mm4 \n\t"\
928 "packssdw %%mm1, %%mm1 \n\t" \
929 "packssdw %%mm6, %%mm6 \n\t" \
930 "movd %%mm1, 32+" #dst " \n\t"\
931 "packssdw %%mm4, %%mm4 \n\t" \
932 "packssdw %%mm5, %%mm5 \n\t" \
933 "movd %%mm6, 48+" #dst " \n\t"\
934 "movd %%mm4, 64+" #dst " \n\t"\
935 "movd %%mm5, 80+" #dst " \n\t"
939 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
940 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
941 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
942 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
948 #define IDCT(src0, src4, src1, src5, dst, shift) \
949 "movq " #src0 ", %%mm0 \n\t" \
950 "movq " #src4 ", %%mm1 \n\t" \
951 "movq 16(%2), %%mm4 \n\t" \
952 "pmaddwd %%mm0, %%mm4 \n\t" \
953 "movq 24(%2), %%mm5 \n\t" \
954 "pmaddwd %%mm5, %%mm0 \n\t" \
955 "movq 32(%2), %%mm5 \n\t" \
956 "pmaddwd %%mm1, %%mm5 \n\t" \
957 "movq 40(%2), %%mm6 \n\t" \
958 "pmaddwd %%mm6, %%mm1 \n\t" \
959 "movq %%mm4, %%mm6 \n\t" \
960 "paddd %%mm5, %%mm4 \n\t" \
961 "psubd %%mm5, %%mm6 \n\t" \
962 "movq %%mm0, %%mm5 \n\t" \
963 "paddd %%mm1, %%mm0 \n\t" \
964 "psubd %%mm1, %%mm5 \n\t" \
965 "movq 8+" #src0 ", %%mm2 \n\t" \
966 "movq 8+" #src4 ", %%mm3 \n\t" \
967 "movq 16(%2), %%mm1 \n\t" \
968 "pmaddwd %%mm2, %%mm1 \n\t" \
969 "movq 24(%2), %%mm7 \n\t" \
970 "pmaddwd %%mm7, %%mm2 \n\t" \
971 "movq 32(%2), %%mm7 \n\t" \
972 "pmaddwd %%mm3, %%mm7 \n\t" \
973 "pmaddwd 40(%2), %%mm3 \n\t" \
974 "paddd %%mm1, %%mm7 \n\t" \
975 "paddd %%mm1, %%mm1 \n\t" \
976 "psubd %%mm7, %%mm1 \n\t" \
977 "paddd %%mm2, %%mm3 \n\t" \
978 "paddd %%mm2, %%mm2 \n\t" \
979 "psubd %%mm3, %%mm2 \n\t" \
980 "psrad $" #shift ", %%mm4 \n\t"\
981 "psrad $" #shift ", %%mm7 \n\t"\
982 "psrad $" #shift ", %%mm3 \n\t"\
983 "packssdw %%mm7, %%mm4 \n\t" \
984 "movq %%mm4, " #dst " \n\t"\
985 "psrad $" #shift ", %%mm0 \n\t"\
986 "packssdw %%mm3, %%mm0 \n\t" \
987 "movq %%mm0, 16+" #dst " \n\t"\
988 "movq %%mm0, 96+" #dst " \n\t"\
989 "movq %%mm4, 112+" #dst " \n\t"\
990 "psrad $" #shift ", %%mm5 \n\t"\
991 "psrad $" #shift ", %%mm6 \n\t"\
992 "psrad $" #shift ", %%mm2 \n\t"\
993 "packssdw %%mm2, %%mm5 \n\t" \
994 "movq %%mm5, 32+" #dst " \n\t"\
995 "psrad $" #shift ", %%mm1 \n\t"\
996 "packssdw %%mm1, %%mm6 \n\t" \
997 "movq %%mm6, 48+" #dst " \n\t"\
998 "movq %%mm6, 64+" #dst " \n\t"\
999 "movq %%mm5, 80+" #dst " \n\t"
1003 IDCT( 0(%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
1005 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
1010 "
# .p2align 4 \n\t"\
1013 #define IDCT(src0, src4, src1, src5, dst, shift) \
1014 "movq " #src0 ", %%mm0 \n\t" \
1015 "movq " #src4 ", %%mm1 \n\t" \
1016 "movq " #src1 ", %%mm2 \n\t" \
1017 "movq 16(%2), %%mm4 \n\t" \
1018 "pmaddwd %%mm0, %%mm4 \n\t" \
1019 "movq 24(%2), %%mm5 \n\t" \
1020 "pmaddwd %%mm5, %%mm0 \n\t" \
1021 "movq 32(%2), %%mm5 \n\t" \
1022 "pmaddwd %%mm1, %%mm5 \n\t" \
1023 "movq 40(%2), %%mm6 \n\t" \
1024 "pmaddwd %%mm6, %%mm1 \n\t" \
1025 "movq %%mm4, %%mm6 \n\t" \
1026 "movq 48(%2), %%mm7 \n\t" \
1027 "pmaddwd %%mm2, %%mm7 \n\t" \
1028 "paddd %%mm5, %%mm4 \n\t" \
1029 "psubd %%mm5, %%mm6 \n\t" \
1030 "movq %%mm0, %%mm5 \n\t" \
1031 "paddd %%mm1, %%mm0 \n\t" \
1032 "psubd %%mm1, %%mm5 \n\t" \
1033 "movq 64(%2), %%mm1 \n\t"\
1034 "pmaddwd %%mm2, %%mm1 \n\t" \
1035 "paddd %%mm4, %%mm7 \n\t" \
1036 "paddd %%mm4, %%mm4 \n\t" \
1037 "psubd %%mm7, %%mm4 \n\t" \
1038 "psrad $" #shift ", %%mm7 \n\t"\
1039 "psrad $" #shift ", %%mm4 \n\t"\
1040 "movq %%mm0, %%mm3 \n\t" \
1041 "paddd %%mm1, %%mm0 \n\t" \
1042 "psubd %%mm1, %%mm3 \n\t" \
1043 "psrad $" #shift ", %%mm0 \n\t"\
1044 "psrad $" #shift ", %%mm3 \n\t"\
1045 "packssdw %%mm7, %%mm7 \n\t" \
1046 "movd %%mm7, " #dst " \n\t"\
1047 "packssdw %%mm0, %%mm0 \n\t" \
1048 "movd %%mm0, 16+" #dst " \n\t"\
1049 "packssdw %%mm3, %%mm3 \n\t" \
1050 "movd %%mm3, 96+" #dst " \n\t"\
1051 "packssdw %%mm4, %%mm4 \n\t" \
1052 "movd %%mm4, 112+" #dst " \n\t"\
1053 "movq 80(%2), %%mm4 \n\t" \
1054 "pmaddwd %%mm2, %%mm4 \n\t" \
1055 "pmaddwd 96(%2), %%mm2 \n\t" \
1056 "movq %%mm5, %%mm3 \n\t" \
1057 "paddd %%mm4, %%mm3 \n\t" \
1058 "psubd %%mm4, %%mm5 \n\t" \
1059 "psrad $" #shift ", %%mm3 \n\t"\
1060 "psrad $" #shift ", %%mm5 \n\t"\
1061 "movq %%mm6, %%mm4 \n\t" \
1062 "paddd %%mm2, %%mm6 \n\t" \
1063 "psubd %%mm2, %%mm4 \n\t" \
1064 "psrad $" #shift ", %%mm6 \n\t"\
1065 "packssdw %%mm3, %%mm3 \n\t" \
1066 "movd %%mm3, 32+" #dst " \n\t"\
1067 "psrad $" #shift ", %%mm4 \n\t"\
1068 "packssdw %%mm6, %%mm6 \n\t" \
1069 "movd %%mm6, 48+" #dst " \n\t"\
1070 "packssdw %%mm4, %%mm4 \n\t" \
1071 "packssdw %%mm5, %%mm5 \n\t" \
1072 "movd %%mm4, 64+" #dst " \n\t"\
1073 "movd %%mm5, 80+" #dst " \n\t"
1077 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
1078 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
1079 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
1080 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1087 #define IDCT(src0, src4, src1, src5, dst, shift) \
1088 "movq " #src0 ", %%mm0 \n\t" \
1089 "movq 16(%2), %%mm4 \n\t" \
1090 "pmaddwd %%mm0, %%mm4 \n\t" \
1091 "movq 24(%2), %%mm5 \n\t" \
1092 "pmaddwd %%mm5, %%mm0 \n\t" \
1093 "psrad $" #shift ", %%mm4 \n\t"\
1094 "psrad $" #shift ", %%mm0 \n\t"\
1095 "movq 8+" #src0 ", %%mm2 \n\t" \
1096 "movq 16(%2), %%mm1 \n\t" \
1097 "pmaddwd %%mm2, %%mm1 \n\t" \
1098 "movq 24(%2), %%mm7 \n\t" \
1099 "pmaddwd %%mm7, %%mm2 \n\t" \
1100 "movq 32(%2), %%mm7 \n\t" \
1101 "psrad $" #shift ", %%mm1 \n\t"\
1102 "packssdw %%mm1, %%mm4 \n\t" \
1103 "movq %%mm4, " #dst " \n\t"\
1104 "psrad $" #shift ", %%mm2 \n\t"\
1105 "packssdw %%mm2, %%mm0 \n\t" \
1106 "movq %%mm0, 16+" #dst " \n\t"\
1107 "movq %%mm0, 96+" #dst " \n\t"\
1108 "movq %%mm4, 112+" #dst " \n\t"\
1109 "movq %%mm0, 32+" #dst " \n\t"\
1110 "movq %%mm4, 48+" #dst " \n\t"\
1111 "movq %%mm4, 64+" #dst " \n\t"\
1112 "movq %%mm0, 80+" #dst " \n\t"
1115 IDCT( 0(%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
1117 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)