Source code
Revision control
Copy as Markdown
Other Tools
; Copyright © 2022, VideoLAN and dav1d authors
; Copyright © 2022, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
; list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
; this list of conditions and the following disclaimer in the documentation
; and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%include "config.asm"
%include "ext/x86/x86inc.asm"
%if ARCH_X86_64
SECTION_RODATA 64
cdef_perm: db 2, 18, 16, 18, 24, 19, 0, 19, 25, 20, 1, 20, 26, 21, 2, 21
db 3, 26, 3, 26, 28, 27, 4, 27, 29, 28, -1, 28, 30, 29, -1, 29
db 0, 34, 17, 34, 16, 35, 8, 35, 17, 36, 9, 36, 18, 37, 10, 37
db 1, 42, 11, 42, 20, 43, 12, 43, 21, 44, -1, 44, 22, 45, -1, 45
end_perm4: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30
db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62
edge_mask4: dw 0xff99, 0xff88, 0xff11, 0xff00 ; 0100, 0101, 0110, 0111
dw 0x99ff, 0x88ff, 0x11ff, 0x00ff ; 1000, 1001, 1010, 1011
dw 0x9999, 0x8888, 0x1111, 0x0000 ; 1100, 1101, 1110, 1111
pri_taps4: dw 64, 32, 48, 48 ; left-shifted by 4
cdef_dirs4: dw 8, 16, 8, 15, -7,-14, 1, -6
dw 1, 2, 1, 10, 9, 18, 8, 17
dw 8, 16, 8, 15, -7,-14, 1, -6
deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15
cdef_dirs8: db 32, 64, 32, 62,-30,-60, 2,-28
db 2, 4, 2, 36, 34, 68, 32, 66
db 32, 64, 32, 62,-30,-60, 2,-28
pri_taps8: dw 4, 4, 2, 2, 3, 3, 3, 3
sec_taps4: dw 32, 16
pw_m16384: times 2 dw -16384
pw_2048: times 2 dw 2048
pd_268435568: dd 268435568 ; (1 << 28) + (7 << 4)
edge_mask8: dw 0x2121, 0x2020, 0x0101
SECTION .text
%macro CONSTRAIN 7 ; dst, p, px, zero, tresh, shift, tmp
psubw %1, %2, %3
pabsw %1, %1
vpcmpgtw k1, %3, %2
vpsrlvw %7, %1, %6
psubusw %7, %5, %7
pminsw %1, %7
vpsubw %1{k1}, %4, %1
%endmacro
; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25
; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35
; L0 L1 00 01 02 03 04 05 b0 b1 b2 b3 b4 b5 b6 b7
; L2 L3 10 11 12 13 14 15 B0 B1 B2 B3 B4 B5 B6 B7
INIT_ZMM avx512icl
cglobal cdef_filter_4x4_16bpc, 5, 7, 16, dst, stride, left, top, bot, \
pri, sec, dir, damping, edge
%define base r6-cdef_dirs4
lea r6, [cdef_dirs4]
movu xm3, [dstq+strideq*0]
vinserti32x4 ym3, [dstq+strideq*1], 1
mova xm2, [leftq]
lea r2, [dstq+strideq*2]
vinserti32x4 m3, [r2+strideq*0], 2
mova m5, [base+cdef_perm]
vinserti32x4 m3, [r2+strideq*1], 3
vpermt2d m2, m5, m3
vinserti32x4 m1, m2, [topq+strideq*0-4], 0
vinserti32x4 m1, [topq+strideq*1-4], 1
mov r3d, edgem
movifnidn prid, prim
punpcklwd m3, m3 ; px
psrlw m5, 8
vpbroadcastd m0, [base+pd_268435568]
pxor m12, m12
cmp r3d, 0x0f
jne .mask_edges
vinserti32x4 m2, [botq+strideq*0-4], 2
vinserti32x4 m2, [botq+strideq*1-4], 3
.main:
test prid, prid
jz .sec_only
lzcnt r4d, prid
rorx r3d, prid, 2
vpbroadcastw m13, prim
cmp dword r10m, 0xfff ; if (bpc == 12)
cmove prid, r3d ; pri >>= 2
mov r3d, dampingm
and prid, 4
sub r3d, 31
vpbroadcastd m15, [base+pri_taps4+priq]
xor prid, prid
add r4d, r3d
cmovns prid, r4d ; pri_shift
mov r4d, dirm
vpbroadcastw m14, prid
mov r5d, secm
vpbroadcastd m9, [base+cdef_dirs4+(r4+2)*4]
call .constrain
test r5d, r5d
jz .end_no_clip
lzcnt r5d, r5d
vpbroadcastw m13, secm
add r3d, r5d
pminuw m6, m3, m8
pmaxsw m7, m3, m8
pminuw m6, m9
pmaxsw m7, m9
call .constrain_sec
pminuw m6, m8
pmaxsw m7, m8
pminuw m6, m9
pmaxsw m7, m9
vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4]
call .constrain
pminuw m6, m8
pmaxsw m7, m8
pminuw m6, m9
pmaxsw m7, m9
psrldq m8, m6, 2
vpshldd m3, m0, 8
psrldq m9, m7, 2
paddd m0, m3
pminuw m6, m8
psrldq m0, 1
pmaxsw m7, m9
pmaxsw m0, m6
pminsw m0, m7
vpmovdw ym0, m0
jmp .end
.sec_only:
tzcnt r5d, secm
mov r3d, dampingm
vpbroadcastw m13, secm
mov r4d, dirm
sub r3d, r5d ; sec_shift
call .constrain_sec
vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4]
call .constrain
.end_no_clip:
mova ym1, [base+end_perm4]
vpshldd m3, m0, 8 ; (px << 8) + ((sum > -8) << 4)
paddd m0, m3 ; (px << 8) + ((sum + (sum > -8) + 7) << 4)
vpermb m0, m1, m0
.end:
movq [dstq+strideq*0], xm0
movhps [dstq+strideq*1], xm0
vextracti32x4 xm0, ym0, 1
movq [r2+strideq*0], xm0
movhps [r2+strideq*1], xm0
RET
.mask_edges:
vpbroadcastd m6, [base+pw_m16384]
test r3b, 0x08
jz .mask_edges_no_bottom ; avoid buffer overread
vinserti32x4 m2, [botq+strideq*0-4], 2
vinserti32x4 m2, [botq+strideq*1-4], 3
kmovw k1, [base+edge_mask4-8+r3*2]
jmp .mask_edges_main
.mask_edges_no_bottom:
kmovw k1, [base+edge_mask4+8+r3*2]
.mask_edges_main:
or r3d, 0x04
vmovdqa32 m1{k1}, m6 ; edge pixels = -16384
kmovw k1, [base+edge_mask4-8+r3*2]
vmovdqa32 m2{k1}, m6
jmp .main
.constrain_sec:
vpbroadcastd m9, [base+cdef_dirs4+(r4+4)*4]
vpbroadcastw m14, r3d
vpbroadcastd m15, [base+sec_taps4]
.constrain:
paddw m8, m5, m9
vpermi2w m8, m1, m2 ; k0p0 k1p0
psubw m9, m5, m9
vpermi2w m9, m1, m2 ; k0p1 k1p1
CONSTRAIN m10, m8, m3, m12, m13, m14, m11
vpdpwssd m0, m10, m15
CONSTRAIN m10, m9, m3, m12, m13, m14, m11
vpdpwssd m0, m10, m15
ret
; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25 Lc Ld 60 61 62 63 64 65
; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35 Le Lf 70 71 72 73 74 75
; L0 L1 00 01 02 03 04 05 L8 L9 40 41 42 43 44 45 b0 b1 b2 b3 b4 b5 b6 b7
; L2 L3 10 11 12 13 14 15 La Lb 50 51 52 53 54 55 B0 B1 B2 B3 B4 B5 B6 B7
cglobal cdef_filter_4x8_16bpc, 5, 7, 22, dst, stride, left, top, bot, \
pri, sec, dir, damping, edge
lea r6, [cdef_dirs4]
movu xm18, [dstq+strideq*0]
vinserti128 ym18, [dstq+strideq*1], 1
mova xm1, [leftq+16*0]
mova xm2, [leftq+16*1]
lea r2, [strideq*3]
vinserti32x4 m18, [dstq+strideq*2], 2
mova m5, [base+cdef_perm]
vinserti32x4 m18, [dstq+r2 ], 3
vpermt2d m1, m5, m18
vinserti32x4 m0, m1, [topq+strideq*0-4], 0
vinserti32x4 m0, [topq+strideq*1-4], 1
lea r3, [dstq+strideq*4]
movu xm19, [r3+strideq*0]
vinserti128 ym19, [r3+strideq*1], 1
vinserti32x4 m19, [r3+strideq*2], 2
vinserti32x4 m19, [r3+r2 ], 3
mov r3d, edgem
movifnidn prid, prim
vpermt2d m2, m5, m19
vpbroadcastd m16, [base+pd_268435568]
pxor m12, m12
punpcklwd m18, m18 ; px (top)
psrlw m5, 8
punpcklwd m19, m19 ; px (bottom)
mova m17, m16
vshufi32x4 m1, m2, q3210
cmp r3d, 0x0f
jne .mask_edges
vinserti32x4 m2, [botq+strideq*0-4], 2
vinserti32x4 m2, [botq+strideq*1-4], 3
.main:
test prid, prid
jz .sec_only
lzcnt r4d, prid
rorx r3d, prid, 2
vpbroadcastw m13, prim
cmp dword r10m, 0xfff ; if (bpc == 12)
cmove prid, r3d ; pri >>= 2
mov r3d, dampingm
and prid, 4
sub r3d, 31
vpbroadcastd m15, [base+pri_taps4+priq]
xor prid, prid
add r4d, r3d
cmovns prid, r4d ; pri_shift
mov r4d, dirm
vpbroadcastw m14, prid
mov r5d, secm
vpbroadcastd m9, [base+cdef_dirs4+(r4+2)*4]
call .constrain
test r5d, r5d
jz .end_no_clip
lzcnt r5d, r5d
vpbroadcastw m13, secm
add r3d, r5d
pminuw m3, m18, m6
pmaxsw m4, m18, m6
pminuw m20, m19, m7
pmaxsw m21, m19, m7
pminuw m3, m8
pmaxsw m4, m8
pminuw m20, m9
pmaxsw m21, m9
call .constrain_sec
pminuw m3, m6
pmaxsw m4, m6
pminuw m20, m7
pmaxsw m21, m7
pminuw m3, m8
pmaxsw m4, m8
pminuw m20, m9
pmaxsw m21, m9
vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4]
call .constrain
pminuw m3, m6
pmaxsw m4, m6
mov r3, 0xcccccccccccccccc
pminuw m20, m7
pmaxsw m21, m7
kmovq k1, r3
pminuw m3, m8
pmaxsw m4, m8
pminuw m20, m9
pmaxsw m21, m9
vbroadcasti32x4 m0, [base+deint_shuf]
vpshldd m6, m20, m3, 16
vmovdqu8 m3{k1}, m20
vpshldd m18, m16, 8
vpshldd m7, m21, m4, 16
vmovdqu8 m4{k1}, m21
vpshldd m19, m17, 8
pminuw m3, m6
paddd m16, m18
pmaxsw m4, m7
paddd m17, m19
psrldq m16, 1
palignr m16{k1}, m17, m17, 15
lea r6, [dstq+strideq*4]
pmaxsw m16, m3
pminsw m16, m4
pshufb m16, m0
movq [dstq+strideq*0], xm16
movhps [r6 +strideq*0], xm16
vextracti128 xm17, ym16, 1
movq [dstq+strideq*1], xm17
movhps [r6 +strideq*1], xm17
vextracti32x4 xm17, m16, 2
movq [dstq+strideq*2], xm17
movhps [r6 +strideq*2], xm17
vextracti32x4 xm16, m16, 3
movq [dstq+r2 ], xm16
movhps [r6 +r2 ], xm16
RET
.sec_only:
mov r4d, dirm
tzcnt r5d, secm
mov r3d, dampingm
vpbroadcastw m13, secm
sub r3d, r5d ; sec_shift
call .constrain_sec
vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4]
call .constrain
.end_no_clip:
mova ym20, [base+end_perm4]
vpshldd m18, m16, 8 ; (px << 8) + ((sum > -8) << 4)
vpshldd m19, m17, 8
paddd m16, m18 ; (px << 8) + ((sum + (sum > -8) + 7) << 4)
paddd m17, m19
vpermb m16, m20, m16
vpermb m17, m20, m17
movq [dstq+strideq*0], xm16
movhps [dstq+strideq*1], xm16
vextracti128 xm16, ym16, 1
movq [dstq+strideq*2], xm16
movhps [dstq+r2 ], xm16
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm17
movhps [dstq+strideq*1], xm17
vextracti128 xm17, ym17, 1
movq [dstq+strideq*2], xm17
movhps [dstq+r2 ], xm17
RET
.mask_edges:
vpbroadcastd m6, [base+pw_m16384]
test r3b, 0x08
jz .mask_edges_no_bottom ; avoid buffer overread
vinserti32x4 m2, [botq+strideq*0-4], 2
vinserti32x4 m2, [botq+strideq*1-4], 3
kmovw k1, [base+edge_mask4-8+r3*2]
jmp .mask_edges_main
.mask_edges_no_bottom:
kmovw k1, [base+edge_mask4+8+r3*2]
.mask_edges_main:
mov r4d, r3d
or r3d, 0x0c
vmovdqa32 m0{k1}, m6 ; edge pixels = -16384
kmovw k1, [base+edge_mask4-8+r3*2]
or r4d, 0x04
vmovdqa32 m1{k1}, m6
kmovw k1, [base+edge_mask4-8+r4*2]
vmovdqa32 m2{k1}, m6
jmp .main
.constrain_sec:
vpbroadcastd m9, [base+cdef_dirs4+(r4+4)*4]
vpbroadcastw m14, r3d
vpbroadcastd m15, [base+sec_taps4]
.constrain:
paddw m7, m5, m9
mova m6, m0
vpermt2w m6, m7, m1 ; k0p0 k1p0 (top)
psubw m9, m5, m9
mova m8, m0
vpermi2w m7, m1, m2 ; k0p0 k1p0 (bottom)
CONSTRAIN m10, m6, m18, m12, m13, m14, m11
vpermt2w m8, m9, m1 ; k0p1 k1p1 (top)
vpdpwssd m16, m10, m15
CONSTRAIN m10, m7, m19, m12, m13, m14, m11
vpermi2w m9, m1, m2 ; k0p1 k1p1 (bottom)
vpdpwssd m17, m10, m15
CONSTRAIN m10, m8, m18, m12, m13, m14, m11
vpdpwssd m16, m10, m15
CONSTRAIN m10, m9, m19, m12, m13, m14, m11
vpdpwssd m17, m10, m15
ret
cglobal cdef_filter_8x8_16bpc, 5, 7, 22, 64*6, dst, stride, left, top, bot, \
pri, sec, dir, damping, edge
%define base r6-cdef_dirs8
lea r6, [cdef_dirs8]
movu ym17, [dstq+strideq*0]
vinserti32x8 m17, [dstq+strideq*1], 1
movq xm4, [leftq+8*0]
movq xm5, [leftq+8*1]
psrld m2, [base+cdef_perm], 16
movq xm6, [leftq+8*2]
movq xm7, [leftq+8*3]
lea r2, [strideq*3]
movu ym16, [topq+strideq*0-4]
vinserti32x8 m16, [topq+strideq*1-4], 1
lea r3, [dstq+strideq*4]
movu ym18, [dstq+strideq*2]
vinserti32x8 m18, [dstq+r2 ], 1
movu ym19, [r3+strideq*0]
vinserti32x8 m19, [r3+strideq*1], 1
movu ym20, [r3+strideq*2]
vinserti32x8 m20, [r3+r2 ], 1
vshufi32x4 m0, m17, m18, q2020 ; px (top)
mov r3d, edgem
vshufi32x4 m1, m19, m20, q2020 ; px (bottom)
movifnidn prid, prim
vpermt2d m17, m2, m4
vpermt2d m18, m2, m5
pxor m12, m12
vpermt2d m19, m2, m6
vpermt2d m20, m2, m7
cmp r3d, 0x0f
jne .mask_edges
movu ym21, [botq+strideq*0-4]
vinserti32x8 m21, [botq+strideq*1-4], 1
.main:
mova [rsp+64*0], m16 ; top
mova [rsp+64*1], m17 ; 0 1
mova [rsp+64*2], m18 ; 2 3
mova [rsp+64*3], m19 ; 4 5
mova [rsp+64*4], m20 ; 6 7
mova [rsp+64*5], m21 ; bottom
test prid, prid
jz .sec_only
lzcnt r4d, prid
rorx r3d, prid, 2
vpbroadcastw m13, prim
cmp dword r10m, 0xfff ; if (bpc == 12)
cmove prid, r3d ; pri >>= 2
mov r3d, dampingm
and prid, 4
sub r3d, 31
add r4d, r3d ; pri_shift
vpbroadcastw m14, r4d
mov r4d, dirm
vpbroadcastd m2, [base+pri_taps8+priq*2+0]
vpbroadcastd m3, [base+pri_taps8+priq*2+4]
movsx r5, byte [base+cdef_dirs8+(r4+2)*2+0] ; k0off1
pmaxsw m14, m12
call .constrain
mov r5d, secm
pmullw m16, m8, m2
pmullw m17, m9, m2
test r5d, r5d
jnz .pri_sec
movsx r5, byte [base+cdef_dirs8+(r4+2)*2+1] ; k1off1
call .constrain
pmullw m8, m3
pmullw m9, m3
jmp .end_no_clip
.pri_sec:
lzcnt r5d, r5d
add r3d, r5d ; sec_shift
movsx r5, byte [base+cdef_dirs8+(r4+2)*2+1] ; k1off1
pminuw m18, m0, m4
pmaxsw m19, m0, m4
pminuw m20, m1, m5
pmaxsw m21, m1, m5
call .min_max_constrain2
movsx r5, byte [base+cdef_dirs8+(r4+0)*2+0] ; k0off2
pmullw m8, m3
pmullw m9, m3
vpbroadcastw m13, secm
vpbroadcastw m14, r3d
paddw m16, m8
paddw m17, m9
call .min_max_constrain
movsx r5, byte [base+cdef_dirs8+(r4+4)*2+0] ; k0off3
mova m2, m8
mova m3, m9
call .min_max_constrain
movsx r5, byte [base+cdef_dirs8+(r4+0)*2+1] ; k1off2
paddw m2, m8
paddw m3, m9
call .min_max_constrain
movsx r5, byte [base+cdef_dirs8+(r4+4)*2+1] ; k1off3
paddw m2, m2
paddw m3, m3
paddw m16, m8
paddw m17, m9
call .min_max_constrain
vpbroadcastd m10, [base+pw_2048]
paddw m16, m2
paddw m17, m3
paddw m16, m8
paddw m17, m9
psraw m8, m16, 15
psraw m9, m17, 15
paddw m16, m8
paddw m17, m9
pmulhrsw m16, m10
pmulhrsw m17, m10
pminuw m18, m4
pmaxsw m19, m4
pminuw m20, m5
pmaxsw m21, m5
pminuw m18, m6
pmaxsw m19, m6
pminuw m20, m7
pmaxsw m21, m7
paddw m16, m0
paddw m17, m1
pmaxsw m16, m18
pmaxsw m17, m20
pminsw m16, m19
pminsw m17, m21
jmp .end
.sec_only:
tzcnt r5d, secm
mov r4d, dirm
mov r3d, dampingm
vpbroadcastw m13, secm
sub r3d, r5d
movsx r5, byte [base+cdef_dirs8+(r4+0)*2+0]
vpbroadcastw m14, r3d
call .constrain
movsx r5, byte [base+cdef_dirs8+(r4+4)*2+0]
mova m16, m8
mova m17, m9
call .constrain
movsx r5, byte [base+cdef_dirs8+(r4+0)*2+1]
paddw m16, m8
paddw m17, m9
call .constrain
movsx r5, byte [base+cdef_dirs8+(r4+4)*2+1]
paddw m16, m16
paddw m17, m17
paddw m16, m8
paddw m17, m9
call .constrain
.end_no_clip:
vpbroadcastd m10, [base+pw_2048]
paddw m16, m8
paddw m17, m9
psraw m8, m16, 15
psraw m9, m17, 15
paddw m16, m8
paddw m17, m9
pmulhrsw m16, m10
pmulhrsw m17, m10
paddw m16, m0
paddw m17, m1
.end:
mova [dstq+strideq*0], xm16
vextracti128 [dstq+strideq*1], ym16, 1
vextracti32x4 [dstq+strideq*2], m16, 2
vextracti32x4 [dstq+r2 ], m16, 3
lea dstq, [dstq+strideq*4]
mova [dstq+strideq*0], xm17
vextracti128 [dstq+strideq*1], ym17, 1
vextracti32x4 [dstq+strideq*2], m17, 2
vextracti32x4 [dstq+r2 ], m17, 3
RET
.mask_edges:
vpbroadcastd m2, [base+pw_m16384]
test r3b, 0x08
jz .mask_edges_no_bottom ; avoid buffer overread
movu ym21, [botq+strideq*0-4]
vinserti32x8 m21, [botq+strideq*1-4], 1
jmp .mask_edges_top
.mask_edges_no_bottom:
mova m21, m2
.mask_edges_top:
test r3b, 0x04
jnz .mask_edges_main
mova m16, m2
.mask_edges_main:
and r3d, 0x03
cmp r3d, 0x03
je .main
kmovw k1, [base+edge_mask8+r3*2]
vmovdqa32 m16{k1}, m2 ; edge pixels = -16384
vmovdqa32 m17{k1}, m2
vmovdqa32 m18{k1}, m2
vmovdqa32 m19{k1}, m2
vmovdqa32 m20{k1}, m2
vmovdqa32 m21{k1}, m2
jmp .main
ALIGN function_align
.min_max_constrain:
pminuw m18, m4
pmaxsw m19, m4
pminuw m20, m5
pmaxsw m21, m5
.min_max_constrain2:
pminuw m18, m6
pmaxsw m19, m6
pminuw m20, m7
pmaxsw m21, m7
.constrain:
%define tmp rsp+gprsize+68
movu m4, [tmp+r5+64*0]
vshufi32x4 m4, [tmp+r5+64*1], q2020 ; k0p0 (top)
movu m5, [tmp+r5+64*2]
vshufi32x4 m5, [tmp+r5+64*3], q2020 ; k0p0 (bottom)
neg r5
movu m6, [tmp+r5+64*0]
vshufi32x4 m6, [tmp+r5+64*1], q2020 ; k0p1 (top)
movu m7, [tmp+r5+64*2]
vshufi32x4 m7, [tmp+r5+64*3], q2020 ; k0p1 (bottom)
CONSTRAIN m8, m4, m0, m12, m13, m14, m15
CONSTRAIN m9, m5, m1, m12, m13, m14, m15
CONSTRAIN m10, m6, m0, m12, m13, m14, m15
CONSTRAIN m11, m7, m1, m12, m13, m14, m15
paddw m8, m10
paddw m9, m11
ret
%endif ; ARCH_X86_64