Source code
Revision control
Copy as Markdown
Other Tools
; Copyright © 2020, VideoLAN and dav1d authors
; Copyright © 2020, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
; list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
; this list of conditions and the following disclaimer in the documentation
; and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%include "config.asm"
%include "ext/x86/x86inc.asm"
%if ARCH_X86_64
SECTION_RODATA 64
spel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9
db 32, 33, 34, 35, 34, 35, 36, 37, 36, 37, 38, 39, 38, 39, 40, 41
spel_h_shufC: db 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15, 16, 17
db 40, 41, 42, 43, 42, 43, 44, 45, 44, 45, 46, 47, 46, 47, 48, 49
db 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23, 24, 25
db 48, 49, 50, 51, 50, 51, 52, 53, 52, 53, 54, 55, 54, 55, 56, 57
spel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13
db 36, 37, 38, 39, 38, 39, 40, 41, 40, 41, 42, 43, 42, 43, 44, 45
spel_h_shufD: db 12, 13, 14, 15, 14, 15, 16, 17, 16, 17, 18, 19, 18, 19, 20, 21
db 44, 45, 46, 47, 46, 47, 48, 49, 48, 49, 50, 51, 50, 51, 52, 53
db 20, 21, 22, 23, 22, 23, 24, 25, 24, 25, 26, 27, 26, 27, 28, 29
db 52, 53, 54, 55, 54, 55, 56, 57, 56, 57, 58, 59, 58, 59, 60, 61
spel_v_shuf8: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39
db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31
db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47
spel_v_shuf16: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39
db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47
db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55
db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63
prep_endA: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30
db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62
db 65, 66, 69, 70, 73, 74, 77, 78, 81, 82, 85, 86, 89, 90, 93, 94
db 97, 98,101,102,105,106,109,110,113,114,117,118,121,122,125,126
prep_endB: db 1, 2, 5, 6, 9, 10, 13, 14, 33, 34, 37, 38, 41, 42, 45, 46
db 17, 18, 21, 22, 25, 26, 29, 30, 49, 50, 53, 54, 57, 58, 61, 62
db 65, 66, 69, 70, 73, 74, 77, 78, 97, 98,101,102,105,106,109,110
db 81, 82, 85, 86, 89, 90, 93, 94,113,114,117,118,121,122,125,126
prep_endC: db 1, 2, 5, 6, 9, 10, 13, 14, 65, 66, 69, 70, 73, 74, 77, 78
db 17, 18, 21, 22, 25, 26, 29, 30, 81, 82, 85, 86, 89, 90, 93, 94
db 33, 34, 37, 38, 41, 42, 45, 46, 97, 98,101,102,105,106,109,110
db 49, 50, 53, 54, 57, 58, 61, 62,113,114,117,118,121,122,125,126
spel_shuf4a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30
db 17, 18, 33, 34, 21, 22, 37, 38, 25, 26, 41, 42, 29, 30, 45, 46
spel_shuf4b: db 18, 19, 33, 34, 22, 23, 37, 38, 26, 27, 41, 42, 30, 31, 45, 46
db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62
spel_shuf8a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30
db 17, 18, 65, 66, 21, 22, 69, 70, 25, 26, 73, 74, 29, 30, 77, 78
db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62
db 49, 50, 97, 98, 53, 54,101,102, 57, 58,105,106, 61, 62,109,110
spel_shuf8b: db 18, 19, 65, 66, 22, 23, 69, 70, 26, 27, 73, 74, 30, 31, 77, 78
db 65, 66, 81, 82, 69, 70, 85, 86, 73, 74, 89, 90, 77, 78, 93, 94
db 50, 51, 97, 98, 54, 55,101,102, 58, 59,105,106, 62, 63,109,110
db 97, 98,113,114,101,102,117,118,105,106,121,122,109,110,125,126
spel_shuf16: db 1, 2, 33, 34, 5, 6, 37, 38, 9, 10, 41, 42, 13, 14, 45, 46
db 17, 18, 49, 50, 21, 22, 53, 54, 25, 26, 57, 58, 29, 30, 61, 62
db 65, 66, 97, 98, 69, 70,101,102, 73, 74,105,106, 77, 78,109,110
db 81, 82,113,114, 85, 86,117,118, 89, 90,121,122, 93, 94,125,126
spel_shuf32: db 1, 2, 65, 66, 5, 6, 69, 70, 9, 10, 73, 74, 13, 14, 77, 78
db 17, 18, 81, 82, 21, 22, 85, 86, 25, 26, 89, 90, 29, 30, 93, 94
db 33, 34, 97, 98, 37, 38,101,102, 41, 42,105,106, 45, 46,109,110
db 49, 50,113,114, 53, 54,117,118, 57, 58,121,122, 61, 62,125,126
spel_h_shuf2b: db 1, 2, 17, 18, 5, 6, 21, 22, 17, 18, 33, 34, 21, 22, 37, 38
db 33, 34, 49, 50, 37, 38, 53, 54, 49, 50, 9, 10, 53, 54, 13, 14
db 9, 10, 25, 26, 13, 14, 29, 30, 25, 26, 41, 42, 29, 30, 45, 46
spel_shuf2: db 10, 11, 17, 18, 14, 15, 21, 22, 17, 18, 25, 26, 21, 22, 29, 30
spel_h_shuf2a: db 0, 1, 2, 3, 2, 3, 4, 5, 16, 17, 18, 19, 18, 19, 20, 21
db 4, 5, 6, 7, 6, 7, 8, 9, 20, 21, 22, 23, 22, 23, 24, 25
w_mask_end42x: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
w_mask_end444: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94
db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126
w_mask_shuf4: db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30
db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62
db 64, 66, 72, 74, 68, 70, 76, 78, 80, 82, 88, 90, 84, 86, 92, 94
db 96, 98,104,106,100,102,108,110,112,114,120,122,116,118,124,126
w_mask_shuf8: db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30
db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62
db 64, 66, 80, 82, 68, 70, 84, 86, 72, 74, 88, 90, 76, 78, 92, 94
db 96, 98,112,114,100,102,116,118,104,106,120,122,108,110,124,126
w_mask_shuf16: db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46
db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62
db 64, 66, 96, 98, 68, 70,100,102, 72, 74,104,106, 76, 78,108,110
db 80, 82,112,114, 84, 86,116,118, 88, 90,120,122, 92, 94,124,126
warp8x8_permA: db 0, 1, 2, 3, 32, 33, 34, 35, 2, 3, 4, 5, 34, 35, 36, 37
db 4, 5, 6, 7, 36, 37, 38, 39, 6, 7, 8, 9, 38, 39, 40, 41
db 8, 9, 10, 11, 40, 41, 42, 43, 10, 11, 12, 13, 42, 43, 44, 45
db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49
warp8x8_permB: db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49
db 16, 17, 18, 19, 48, 49, 50, 51, 18, 19, 20, 21, 50, 51, 52, 53
db 20, 21, 22, 23, 52, 53, 54, 55, 22, 23, 24, 25, 54, 55, 56, 57
db 24, 25, 26, 27, 56, 57, 58, 59, 26, 27, 28, 29, 58, 59, 60, 61
warp8x8_end: db 0, 1, 4, 5, 16, 17, 20, 21, 32, 33, 36, 37, 48, 49, 52, 53
db 2, 3, 6, 7, 18, 19, 22, 23, 34, 35, 38, 39, 50, 51, 54, 55
db 8, 9, 12, 13, 24, 25, 28, 29, 40, 41, 44, 45, 56, 57, 60, 61
db 10, 11, 14, 15, 26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62, 63
deint_q_shuf: ;dq 0, 2, 4, 6, 1, 3, 5, 7
pd_0to7: dd 0, 1, 2, 3, 4, 5, 6, 7
dd 1
pw_2048: times 2 dw 2048
dd 3
pw_8192: times 2 dw 8192
avg_shift: dw 5, 5, 3, 3
pw_27615: times 2 dw 27615
pw_32766: times 2 dw 32766
warp8x8_permC: db -1, 0, -1, 1, -1, 8, -1, 9, -1, 4, -1, 5, -1, 12, -1, 13
warp8x8_permD: db -1, 2, -1, 3, -1, 10, -1, 11, -1, 6, -1, 7, -1, 14, -1, 15
warp_shift_h: db 11, 19, 11, 19, 43, 51, 43, 51, 13, 21, 13, 21, 45, 53, 45, 53
blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
resize_permA: dd 0, 4, 8, 12, 1, 5, 9, 13, 16, 20, 24, 28, 17, 21, 25, 29
resize_permB: dd 2, 6, 10, 14, 3, 7, 11, 15, 18, 22, 26, 30, 19, 23, 27, 31
resize_permC: dq 0, 1, 4, 5, 8, 9, 12, 13
resize_permD: dq 2, 3, 6, 7, 10, 11, 14, 15
resize_permE: dq 0, 2, 4, 6
resize_shufA: db -1, 0, -1, 1, -1, 4, -1, 5, -1, 8, -1, 9, -1, 12, -1, 13
resize_shufB: db -1, 2, -1, 3, -1, 6, -1, 7, -1, 10, -1, 11, -1, 14, -1, 15
rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7
db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15
prep_hv_shift: dq 6, 4
put_bilin_h_rnd: dw 8, 8, 10, 10
prep_mul: dw 16, 16, 4, 4
put_8tap_h_rnd: dd 34, 40
prep_8tap_rnd: dd 128 - (8192 << 8)
warp_8x8_rnd_h: dd 512, 2048
warp_8x8_rnd_v: dd 262144, 65536
warp_8x8t_rnd_v: dd 16384 - (8192 << 15)
avg_round: dw -16400, -16400, -16388, -16388
w_avg_round: dd 128 + (8192 << 4), 32 + (8192 << 4)
mask_round: dd 512 + (8192 << 6), 128 + (8192 << 6)
w_mask_round: dd 128, 64
bidir_shift: dw 6, 6, 4, 4
pb_64: times 4 db 64
pw_m512: times 2 dw -512
pw_2: times 2 dw 2
pw_64: times 2 dw 64
pd_32: dd 32
pd_63: dd 63
pd_128: dd 128
pd_640: dd 640
pd_2176: dd 2176
pd_16384: dd 16384
pd_0_4: dd 0, 4
%define pw_16 prep_mul
%define pd_512 warp_8x8_rnd_h
%macro BASE_JMP_TABLE 3-*
%xdefine %1_%2_table (%%table - %3)
%xdefine %%base %1_%2
%%table:
%rep %0 - 2
dw %%base %+ _w%3 - %%base
%rotate 1
%endrep
%endmacro
%macro HV_JMP_TABLE 5-*
%xdefine %%prefix mangle(private_prefix %+ _%1_%2_16bpc_%3)
%xdefine %%base %1_%3
%assign %%types %4
%if %%types & 1
%xdefine %1_%2_h_%3_table (%%h - %5)
%%h:
%rep %0 - 4
dw %%prefix %+ .h_w%5 - %%base
%rotate 1
%endrep
%rotate 4
%endif
%if %%types & 2
%xdefine %1_%2_v_%3_table (%%v - %5)
%%v:
%rep %0 - 4
dw %%prefix %+ .v_w%5 - %%base
%rotate 1
%endrep
%rotate 4
%endif
%if %%types & 4
%xdefine %1_%2_hv_%3_table (%%hv - %5)
%%hv:
%rep %0 - 4
dw %%prefix %+ .hv_w%5 - %%base
%rotate 1
%endrep
%endif
%endmacro
%macro BIDIR_JMP_TABLE 2-*
%xdefine %1_%2_table (%%table - 2*%3)
%xdefine %%base %1_%2_table
%xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2)
%%table:
%rep %0 - 2
dd %%prefix %+ .w%3 - %%base
%rotate 1
%endrep
%endmacro
%xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_16bpc_avx512icl.put)
%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_16bpc_avx512icl.prep)
BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE blend, avx512icl, 4, 8, 16, 32
BIDIR_JMP_TABLE blend_v, avx512icl, 2, 4, 8, 16, 32
BIDIR_JMP_TABLE blend_h, avx512icl, 2, 4, 8, 16, 32, 64, 128
BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128
BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE put, 8tap, avx512icl, 2, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, 8tap, avx512icl, 2, 4, 8, 16, 32, 64, 128
%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX
cextern mc_subpel_filters
%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
cextern mc_warp_filter
cextern obmc_masks_avx2
cextern resize_filter
SECTION .text
%if WIN64
DECLARE_REG_TMP 4
%else
DECLARE_REG_TMP 8
%endif
INIT_ZMM avx512icl
cglobal put_bilin_16bpc, 4, 8, 13, dst, ds, src, ss, w, h, mxy
mov mxyd, r6m ; mx
lea r7, [put_avx512icl]
tzcnt t0d, wm
movifnidn hd, hm
test mxyd, mxyd
jnz .h
mov mxyd, r7m ; my
test mxyd, mxyd
jnz .v
.put:
movzx t0d, word [r7+t0*2+table_offset(put,)]
add t0, r7
jmp t0
.put_w2:
mov r6d, [srcq+ssq*0]
mov r7d, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mov [dstq+dsq*0], r6d
mov [dstq+dsq*1], r7d
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w2
RET
.put_w4:
mov r6, [srcq+ssq*0]
mov r7, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mov [dstq+dsq*0], r6
mov [dstq+dsq*1], r7
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w4
RET
.put_w8:
movu xmm0, [srcq+ssq*0]
movu xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0], xmm0
mova [dstq+dsq*1], xmm1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w8
RET
.put_w16:
movu ym0, [srcq+ssq*0]
movu ym1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0], ym0
mova [dstq+dsq*1], ym1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w16
RET
.put_w32:
movu m0, [srcq+ssq*0]
movu m1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0], m0
mova [dstq+dsq*1], m1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w32
RET
.put_w64:
movu m0, [srcq+ssq*0+64*0]
movu m1, [srcq+ssq*0+64*1]
movu m2, [srcq+ssq*1+64*0]
movu m3, [srcq+ssq*1+64*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0+64*0], m0
mova [dstq+dsq*0+64*1], m1
mova [dstq+dsq*1+64*0], m2
mova [dstq+dsq*1+64*1], m3
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w64
RET
.put_w128:
movu m0, [srcq+64*0]
movu m1, [srcq+64*1]
movu m2, [srcq+64*2]
movu m3, [srcq+64*3]
add srcq, ssq
mova [dstq+64*0], m0
mova [dstq+64*1], m1
mova [dstq+64*2], m2
mova [dstq+64*3], m3
add dstq, dsq
dec hd
jg .put_w128
RET
.h:
vpbroadcastw m5, mxyd
mov mxyd, r7m ; my
vpbroadcastd m4, [pw_16]
psubw m4, m5
test mxyd, mxyd
jnz .hv
; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v
movzx t0d, word [r7+t0*2+table_offset(put, _bilin_h)]
mov r6d, r8m ; bitdepth_max
add t0, r7
shr r6d, 11
vpbroadcastd m6, [r7-put_avx512icl+put_bilin_h_rnd+r6*4]
jmp t0
.h_w2:
movq xmm1, [srcq+ssq*0]
movhps xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pmullw xmm0, xmm1, xm4
psrlq xmm1, 16
pmullw xmm1, xm5
paddw xmm0, xm6
paddw xmm0, xmm1
psrlw xmm0, 4
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w2
RET
.h_w4:
movq xmm0, [srcq+ssq*0+0]
movhps xmm0, [srcq+ssq*1+0]
movq xmm1, [srcq+ssq*0+2]
movhps xmm1, [srcq+ssq*1+2]
lea srcq, [srcq+ssq*2]
pmullw xmm0, xm4
pmullw xmm1, xm5
paddw xmm0, xm6
paddw xmm0, xmm1
psrlw xmm0, 4
movq [dstq+dsq*0], xmm0
movhps [dstq+dsq*1], xmm0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w4
RET
.h_w8:
movu xm0, [srcq+ssq*0+0]
vinserti32x4 ym0, [srcq+ssq*1+0], 1
movu xm1, [srcq+ssq*0+2]
vinserti32x4 ym1, [srcq+ssq*1+2], 1
lea srcq, [srcq+ssq*2]
pmullw ym0, ym4
pmullw ym1, ym5
paddw ym0, ym6
paddw ym0, ym1
psrlw ym0, 4
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], ym0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w8
RET
.h_w16:
movu ym0, [srcq+ssq*0+0]
vinserti32x8 m0, [srcq+ssq*1+0], 1
movu ym1, [srcq+ssq*0+2]
vinserti32x8 m1, [srcq+ssq*1+2], 1
lea srcq, [srcq+ssq*2]
pmullw m0, m4
pmullw m1, m5
paddw m0, m6
paddw m0, m1
psrlw m0, 4
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w16
RET
.h_w32:
pmullw m0, m4, [srcq+ssq*0+0]
pmullw m2, m5, [srcq+ssq*0+2]
pmullw m1, m4, [srcq+ssq*1+0]
pmullw m3, m5, [srcq+ssq*1+2]
lea srcq, [srcq+ssq*2]
paddw m0, m6
paddw m1, m6
paddw m0, m2
paddw m1, m3
psrlw m0, 4
psrlw m1, 4
mova [dstq+dsq*0], m0
mova [dstq+dsq*1], m1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w32
RET
.h_w64:
pmullw m0, m4, [srcq+64*0+0]
pmullw m2, m5, [srcq+64*0+2]
pmullw m1, m4, [srcq+64*1+0]
pmullw m3, m5, [srcq+64*1+2]
add srcq, ssq
paddw m0, m6
paddw m1, m6
paddw m0, m2
paddw m1, m3
psrlw m0, 4
psrlw m1, 4
mova [dstq+64*0], m0
mova [dstq+64*1], m1
add dstq, dsq
dec hd
jg .h_w64
RET
.h_w128:
pmullw m0, m4, [srcq+64*0+0]
pmullw m7, m5, [srcq+64*0+2]
pmullw m1, m4, [srcq+64*1+0]
pmullw m8, m5, [srcq+64*1+2]
pmullw m2, m4, [srcq+64*2+0]
pmullw m9, m5, [srcq+64*2+2]
pmullw m3, m4, [srcq+64*3+0]
pmullw m10, m5, [srcq+64*3+2]
add srcq, ssq
REPX {paddw x, m6}, m0, m1, m2, m3
paddw m0, m7
paddw m1, m8
paddw m2, m9
paddw m3, m10
REPX {psrlw x, 4}, m0, m1, m2, m3
mova [dstq+64*0], m0
mova [dstq+64*1], m1
mova [dstq+64*2], m2
mova [dstq+64*3], m3
add dstq, dsq
dec hd
jg .h_w128
RET
.v:
movzx t0d, word [r7+t0*2+table_offset(put, _bilin_v)]
shl mxyd, 11
vpbroadcastw m8, mxyd
add t0, r7
jmp t0
.v_w2:
movd xmm0, [srcq+ssq*0]
.v_w2_loop:
movd xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
punpckldq xmm2, xmm0, xmm1
movd xmm0, [srcq+ssq*0]
punpckldq xmm1, xmm0
psubw xmm1, xmm2
pmulhrsw xmm1, xm8
paddw xmm1, xmm2
movd [dstq+dsq*0], xmm1
pextrd [dstq+dsq*1], xmm1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w2_loop
RET
.v_w4:
movq xmm0, [srcq+ssq*0]
.v_w4_loop:
movq xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
punpcklqdq xmm2, xmm0, xmm1
movq xmm0, [srcq+ssq*0]
punpcklqdq xmm1, xmm0
psubw xmm1, xmm2
pmulhrsw xmm1, xm8
paddw xmm1, xmm2
movq [dstq+dsq*0], xmm1
movhps [dstq+dsq*1], xmm1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w4_loop
RET
.v_w8:
movu xmm0, [srcq+ssq*0]
.v_w8_loop:
vbroadcasti128 ymm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpblendd ymm2, ymm0, ymm1, 0xf0
vbroadcasti128 ymm0, [srcq+ssq*0]
vpblendd ymm1, ymm0, 0xf0
psubw ymm1, ymm2
pmulhrsw ymm1, ym8
paddw ymm1, ymm2
mova [dstq+dsq*0], xmm1
vextracti128 [dstq+dsq*1], ymm1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w8_loop
vzeroupper
RET
.v_w16:
movu ym0, [srcq+ssq*0]
.v_w16_loop:
movu ym3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
psubw ym1, ym3, ym0
pmulhrsw ym1, ym8
paddw ym1, ym0
movu ym0, [srcq+ssq*0]
psubw ym2, ym0, ym3
pmulhrsw ym2, ym8
paddw ym2, ym3
mova [dstq+dsq*0], ym1
mova [dstq+dsq*1], ym2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w16_loop
RET
.v_w32:
movu m0, [srcq+ssq*0]
.v_w32_loop:
movu m3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
psubw m1, m3, m0
pmulhrsw m1, m8
paddw m1, m0
movu m0, [srcq+ssq*0]
psubw m2, m0, m3
pmulhrsw m2, m8
paddw m2, m3
mova [dstq+dsq*0], m1
mova [dstq+dsq*1], m2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w32_loop
RET
.v_w64:
movu m0, [srcq+ssq*0+64*0]
movu m1, [srcq+ssq*0+64*1]
.v_w64_loop:
movu m2, [srcq+ssq*1+64*0]
movu m3, [srcq+ssq*1+64*1]
lea srcq, [srcq+ssq*2]
psubw m4, m2, m0
pmulhrsw m4, m8
paddw m4, m0
movu m0, [srcq+ssq*0+64*0]
psubw m5, m3, m1
pmulhrsw m5, m8
paddw m5, m1
movu m1, [srcq+ssq*0+64*1]
psubw m6, m0, m2
pmulhrsw m6, m8
psubw m7, m1, m3
pmulhrsw m7, m8
mova [dstq+dsq*0+64*0], m4
mova [dstq+dsq*0+64*1], m5
paddw m6, m2
paddw m7, m3
mova [dstq+dsq*1+64*0], m6
mova [dstq+dsq*1+64*1], m7
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w64_loop
RET
.v_w128:
movu m0, [srcq+ssq*0+64*0]
movu m1, [srcq+ssq*0+64*1]
movu m2, [srcq+ssq*0+64*2]
movu m3, [srcq+ssq*0+64*3]
.v_w128_loop:
movu m4, [srcq+ssq*1+64*0]
movu m5, [srcq+ssq*1+64*1]
movu m6, [srcq+ssq*1+64*2]
movu m7, [srcq+ssq*1+64*3]
lea srcq, [srcq+ssq*2]
psubw m9, m4, m0
pmulhrsw m9, m8
paddw m9, m0
movu m0, [srcq+ssq*0+64*0]
psubw m10, m5, m1
pmulhrsw m10, m8
paddw m10, m1
movu m1, [srcq+ssq*0+64*1]
psubw m11, m6, m2
pmulhrsw m11, m8
paddw m11, m2
movu m2, [srcq+ssq*0+64*2]
psubw m12, m7, m3
pmulhrsw m12, m8
paddw m12, m3
movu m3, [srcq+ssq*0+64*3]
mova [dstq+dsq*0+64*0], m9
psubw m9, m0, m4
pmulhrsw m9, m8
mova [dstq+dsq*0+64*1], m10
psubw m10, m1, m5
pmulhrsw m10, m8
mova [dstq+dsq*0+64*2], m11
psubw m11, m2, m6
pmulhrsw m11, m8
mova [dstq+dsq*0+64*3], m12
psubw m12, m3, m7
pmulhrsw m12, m8
paddw m9, m4
paddw m10, m5
mova [dstq+dsq*1+64*0], m9
mova [dstq+dsq*1+64*1], m10
paddw m11, m6
paddw m12, m7
mova [dstq+dsq*1+64*2], m11
mova [dstq+dsq*1+64*3], m12
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w128_loop
RET
.hv:
movzx t0d, word [r7+t0*2+table_offset(put, _bilin_hv)]
shl mxyd, 11
vpbroadcastd m6, [pw_2]
vpbroadcastw m7, mxyd
vpbroadcastd m8, [pw_8192]
add t0, r7
test dword r8m, 0x800
jnz .hv_12bpc
psllw m4, 2
psllw m5, 2
vpbroadcastd m8, [pw_2048]
.hv_12bpc:
jmp t0
.hv_w2:
vpbroadcastq xmm1, [srcq+ssq*0]
pmullw xmm0, xmm1, xm4
psrlq xmm1, 16
pmullw xmm1, xm5
paddw xmm0, xm6
paddw xmm0, xmm1
psrlw xmm0, 2
.hv_w2_loop:
movq xmm2, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
movhps xmm2, [srcq+ssq*0]
pmullw xmm1, xmm2, xm4
psrlq xmm2, 16
pmullw xmm2, xm5
paddw xmm1, xm6
paddw xmm1, xmm2
psrlw xmm1, 2 ; 1 _ 2 _
shufpd xmm2, xmm0, xmm1, 0x01 ; 0 _ 1 _
mova xmm0, xmm1
psubw xmm1, xmm2
paddw xmm1, xmm1
pmulhw xmm1, xm7
paddw xmm1, xmm2
pmulhrsw xmm1, xm8
movd [dstq+dsq*0], xmm1
pextrd [dstq+dsq*1], xmm1, 2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w2_loop
RET
.hv_w4:
pmullw xmm0, xm4, [srcq+ssq*0-8]
pmullw xmm1, xm5, [srcq+ssq*0-6]
paddw xmm0, xm6
paddw xmm0, xmm1
psrlw xmm0, 2
.hv_w4_loop:
movq xmm1, [srcq+ssq*1+0]
movq xmm2, [srcq+ssq*1+2]
lea srcq, [srcq+ssq*2]
movhps xmm1, [srcq+ssq*0+0]
movhps xmm2, [srcq+ssq*0+2]
pmullw xmm1, xm4
pmullw xmm2, xm5
paddw xmm1, xm6
paddw xmm1, xmm2
psrlw xmm1, 2 ; 1 2
shufpd xmm2, xmm0, xmm1, 0x01 ; 0 1
mova xmm0, xmm1
psubw xmm1, xmm2
paddw xmm1, xmm1
pmulhw xmm1, xm7
paddw xmm1, xmm2
pmulhrsw xmm1, xm8
movq [dstq+dsq*0], xmm1
movhps [dstq+dsq*1], xmm1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w4_loop
RET
.hv_w8:
pmullw xmm0, xm4, [srcq+ssq*0+0]
pmullw xmm1, xm5, [srcq+ssq*0+2]
paddw xmm0, xm6
paddw xmm0, xmm1
psrlw xmm0, 2
vinserti32x4 ym0, xmm0, 1
.hv_w8_loop:
movu xm1, [srcq+ssq*1+0]
movu xm2, [srcq+ssq*1+2]
lea srcq, [srcq+ssq*2]
vinserti32x4 ym1, [srcq+ssq*0+0], 1
vinserti32x4 ym2, [srcq+ssq*0+2], 1
pmullw ym1, ym4
pmullw ym2, ym5
paddw ym1, ym6
paddw ym1, ym2
psrlw ym1, 2 ; 1 2
vshufi32x4 ym2, ym0, ym1, 0x01 ; 0 1
mova ym0, ym1
psubw ym1, ym2
paddw ym1, ym1
pmulhw ym1, ym7
paddw ym1, ym2
pmulhrsw ym1, ym8
mova [dstq+dsq*0], xm1
vextracti32x4 [dstq+dsq*1], ym1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w8_loop
RET
.hv_w16:
pmullw ym0, ym4, [srcq+ssq*0+0]
pmullw ym1, ym5, [srcq+ssq*0+2]
paddw ym0, ym6
paddw ym0, ym1
psrlw ym0, 2
vinserti32x8 m0, ym0, 1
.hv_w16_loop:
movu ym1, [srcq+ssq*1+0]
movu ym2, [srcq+ssq*1+2]
lea srcq, [srcq+ssq*2]
vinserti32x8 m1, [srcq+ssq*0+0], 1
vinserti32x8 m2, [srcq+ssq*0+2], 1
pmullw m1, m4
pmullw m2, m5
paddw m1, m6
paddw m1, m2
psrlw m1, 2 ; 1 2
vshufi32x4 m2, m0, m1, q1032 ; 0 1
mova m0, m1
psubw m1, m2
paddw m1, m1
pmulhw m1, m7
paddw m1, m2
pmulhrsw m1, m8
mova [dstq+dsq*0], ym1
vextracti32x8 [dstq+dsq*1], m1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w16_loop
RET
.hv_w32:
.hv_w64:
.hv_w128:
movifnidn wd, wm
lea r6d, [hq+wq*8-256]
mov r4, srcq
mov r7, dstq
.hv_w32_loop0:
pmullw m0, m4, [srcq+ssq*0+0]
pmullw m1, m5, [srcq+ssq*0+2]
paddw m0, m6
paddw m0, m1
psrlw m0, 2
.hv_w32_loop:
pmullw m3, m4, [srcq+ssq*1+0]
pmullw m1, m5, [srcq+ssq*1+2]
lea srcq, [srcq+ssq*2]
paddw m3, m6
paddw m3, m1
psrlw m3, 2
psubw m1, m3, m0
paddw m1, m1
pmulhw m1, m7
paddw m1, m0
pmullw m0, m4, [srcq+ssq*0+0]
pmullw m2, m5, [srcq+ssq*0+2]
paddw m0, m6
paddw m0, m2
psrlw m0, 2
psubw m2, m0, m3
paddw m2, m2
pmulhw m2, m7
paddw m2, m3
pmulhrsw m1, m8
pmulhrsw m2, m8
mova [dstq+dsq*0], m1
mova [dstq+dsq*1], m2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w32_loop
add r4, 64
add r7, 64
movzx hd, r6b
mov srcq, r4
mov dstq, r7
sub r6d, 1<<8
jg .hv_w32_loop0
RET
cglobal prep_bilin_16bpc, 3, 7, 16, tmp, src, stride, w, h, mxy, stride3
movifnidn mxyd, r5m ; mx
lea r6, [prep_avx512icl]
tzcnt wd, wm
movifnidn hd, hm
test mxyd, mxyd
jnz .h
mov mxyd, r6m ; my
test mxyd, mxyd
jnz .v
.prep:
movzx wd, word [r6+wq*2+table_offset(prep,)]
mov r5d, r7m ; bitdepth_max
vpbroadcastd m5, [r6-prep_avx512icl+pw_8192]
add wq, r6
shr r5d, 11
vpbroadcastd m4, [r6-prep_avx512icl+prep_mul+r5*4]
lea stride3q, [strideq*3]
jmp wq
.prep_w4:
movq xmm0, [srcq+strideq*0]
movhps xmm0, [srcq+strideq*1]
vpbroadcastq ymm1, [srcq+strideq*2]
vpbroadcastq ymm2, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vpblendd ymm0, ymm1, 0x30
vpblendd ymm0, ymm2, 0xc0
pmullw ymm0, ym4
psubw ymm0, ym5
mova [tmpq], ymm0
add tmpq, 32
sub hd, 4
jg .prep_w4
vzeroupper
RET
.prep_w8:
movu xm0, [srcq+strideq*0]
vinserti32x4 ym0, [srcq+strideq*1], 1
vinserti32x4 m0, [srcq+strideq*2], 2
vinserti32x4 m0, [srcq+stride3q ], 3
lea srcq, [srcq+strideq*4]
pmullw m0, m4
psubw m0, m5
mova [tmpq], m0
add tmpq, 64
sub hd, 4
jg .prep_w8
RET
.prep_w16:
movu ym0, [srcq+strideq*0]
vinserti32x8 m0, [srcq+strideq*1], 1
movu ym1, [srcq+strideq*2]
vinserti32x8 m1, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
pmullw m0, m4
pmullw m1, m4
psubw m0, m5
psubw m1, m5
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
add tmpq, 64*2
sub hd, 4
jg .prep_w16
RET
.prep_w32:
pmullw m0, m4, [srcq+strideq*0]
pmullw m1, m4, [srcq+strideq*1]
pmullw m2, m4, [srcq+strideq*2]
pmullw m3, m4, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
REPX {psubw x, m5}, m0, m1, m2, m3
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
sub hd, 4
jg .prep_w32
RET
.prep_w64:
pmullw m0, m4, [srcq+strideq*0+64*0]
pmullw m1, m4, [srcq+strideq*0+64*1]
pmullw m2, m4, [srcq+strideq*1+64*0]
pmullw m3, m4, [srcq+strideq*1+64*1]
lea srcq, [srcq+strideq*2]
REPX {psubw x, m5}, m0, m1, m2, m3
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
sub hd, 2
jg .prep_w64
RET
.prep_w128:
pmullw m0, m4, [srcq+64*0]
pmullw m1, m4, [srcq+64*1]
pmullw m2, m4, [srcq+64*2]
pmullw m3, m4, [srcq+64*3]
add srcq, strideq
REPX {psubw x, m5}, m0, m1, m2, m3
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
dec hd
jg .prep_w128
RET
.h:
vpbroadcastw m5, mxyd
mov mxyd, r6m ; my
vpbroadcastd m4, [pw_16]
vpbroadcastd m6, [pw_32766]
psubw m4, m5
test dword r7m, 0x800
jnz .h_12bpc
psllw m4, 2
psllw m5, 2
.h_12bpc:
test mxyd, mxyd
jnz .hv
movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)]
add wq, r6
lea stride3q, [strideq*3]
jmp wq
.h_w4:
movu xm1, [srcq+strideq*0]
vinserti32x4 ym1, [srcq+strideq*2], 1
movu xm2, [srcq+strideq*1]
vinserti32x4 ym2, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
punpcklqdq ym0, ym1, ym2
psrldq ym1, 2
psrldq ym2, 2
pmullw ym0, ym4
punpcklqdq ym1, ym2
pmullw ym1, ym5
psubw ym0, ym6
paddw ym0, ym1
psraw ym0, 2
mova [tmpq], ym0
add tmpq, 32
sub hd, 4
jg .h_w4
RET
.h_w8:
movu xm0, [srcq+strideq*0+0]
movu xm1, [srcq+strideq*0+2]
vinserti32x4 ym0, [srcq+strideq*1+0], 1
vinserti32x4 ym1, [srcq+strideq*1+2], 1
vinserti32x4 m0, [srcq+strideq*2+0], 2
vinserti32x4 m1, [srcq+strideq*2+2], 2
vinserti32x4 m0, [srcq+stride3q +0], 3
vinserti32x4 m1, [srcq+stride3q +2], 3
lea srcq, [srcq+strideq*4]
pmullw m0, m4
pmullw m1, m5
psubw m0, m6
paddw m0, m1
psraw m0, 2
mova [tmpq], m0
add tmpq, 64
sub hd, 4
jg .h_w8
RET
.h_w16:
movu ym0, [srcq+strideq*0+0]
vinserti32x8 m0, [srcq+strideq*1+0], 1
movu ym1, [srcq+strideq*0+2]
vinserti32x8 m1, [srcq+strideq*1+2], 1
lea srcq, [srcq+strideq*2]
pmullw m0, m4
pmullw m1, m5
psubw m0, m6
paddw m0, m1
psraw m0, 2
mova [tmpq], m0
add tmpq, 64
sub hd, 2
jg .h_w16
RET
.h_w32:
pmullw m0, m4, [srcq+strideq*0+0]
pmullw m2, m5, [srcq+strideq*0+2]
pmullw m1, m4, [srcq+strideq*1+0]
pmullw m3, m5, [srcq+strideq*1+2]
lea srcq, [srcq+strideq*2]
psubw m0, m6
psubw m1, m6
paddw m0, m2
paddw m1, m3
psraw m0, 2
psraw m1, 2
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
add tmpq, 64*2
sub hd, 2
jg .h_w32
RET
.h_w64:
pmullw m0, m4, [srcq+ 0]
pmullw m2, m5, [srcq+ 2]
pmullw m1, m4, [srcq+64]
pmullw m3, m5, [srcq+66]
add srcq, strideq
psubw m0, m6
psubw m1, m6
paddw m0, m2
paddw m1, m3
psraw m0, 2
psraw m1, 2
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
add tmpq, 64*2
dec hd
jg .h_w64
RET
.h_w128:
pmullw m0, m4, [srcq+ 0]
pmullw m7, m5, [srcq+ 2]
pmullw m1, m4, [srcq+ 64]
pmullw m8, m5, [srcq+ 66]
pmullw m2, m4, [srcq+128]
pmullw m9, m5, [srcq+130]
pmullw m3, m4, [srcq+192]
pmullw m10, m5, [srcq+194]
add srcq, strideq
REPX {psubw x, m6}, m0, m1, m2, m3
paddw m0, m7
paddw m1, m8
paddw m2, m9
paddw m3, m10
REPX {psraw x, 2}, m0, m1, m2, m3
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
dec hd
jg .h_w128
RET
.v:
movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)]
vpbroadcastw m9, mxyd
vpbroadcastd m8, [pw_16]
vpbroadcastd m10, [pw_32766]
add wq, r6
lea stride3q, [strideq*3]
psubw m8, m9
test dword r7m, 0x800
jnz .v_12bpc
psllw m8, 2
psllw m9, 2
.v_12bpc:
jmp wq
.v_w4:
movq xmm0, [srcq+strideq*0]
.v_w4_loop:
vpbroadcastq xmm2, [srcq+strideq*1]
vpbroadcastq ymm1, [srcq+strideq*2]
vpbroadcastq ymm3, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vpblendd ymm2, ymm1, 0x30
vpblendd ymm2, ymm3, 0xc0
vpblendd ymm1, ymm2, ymm0, 0x03 ; 0 1 2 3
movq xmm0, [srcq+strideq*0]
valignq ymm2, ymm0, ymm2, 1 ; 1 2 3 4
pmullw ymm1, ym8
pmullw ymm2, ym9
psubw ymm1, ym10
paddw ymm1, ymm2
psraw ymm1, 2
mova [tmpq], ymm1
add tmpq, 32
sub hd, 4
jg .v_w4_loop
vzeroupper
RET
.v_w8:
movu xm0, [srcq+strideq*0]
.v_w8_loop:
vinserti32x4 ym1, ym0, [srcq+strideq*1], 1
vinserti32x4 m1, [srcq+strideq*2], 2
vinserti32x4 m1, [srcq+stride3q ], 3 ; 0 1 2 3
lea srcq, [srcq+strideq*4]
movu xm0, [srcq+strideq*0]
valignq m2, m0, m1, 2 ; 1 2 3 4
pmullw m1, m8
pmullw m2, m9
psubw m1, m10
paddw m1, m2
psraw m1, 2
mova [tmpq], m1
add tmpq, 64
sub hd, 4
jg .v_w8_loop
RET
.v_w16:
movu ym0, [srcq+strideq*0]
.v_w16_loop:
vinserti32x8 m1, m0, [srcq+strideq*1], 1 ; 0 1
movu ym3, [srcq+strideq*2]
vinserti32x8 m2, m3, [srcq+stride3q ], 1 ; 2 3
lea srcq, [srcq+strideq*4]
movu ym0, [srcq+strideq*0]
vshufi32x4 m3, m1, m3, q1032 ; 1 2
vshufi32x4 m4, m2, m0, q1032 ; 3 4
pmullw m1, m8
pmullw m2, m8
pmullw m3, m9
pmullw m4, m9
psubw m1, m10
psubw m2, m10
paddw m1, m3
paddw m2, m4
psraw m1, 2
psraw m2, 2
mova [tmpq+64*0], m1
mova [tmpq+64*1], m2
add tmpq, 64*2
sub hd, 4
jg .v_w16_loop
RET
.v_w32:
movu m0, [srcq+strideq*0]
.v_w32_loop:
movu m3, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
pmullw m1, m8, m0
movu m0, [srcq+strideq*0]
pmullw m2, m8, m3
pmullw m3, m9
pmullw m4, m9, m0
psubw m1, m10
psubw m2, m10
paddw m1, m3
paddw m2, m4
psraw m1, 2
psraw m2, 2
mova [tmpq+64*0], m1
mova [tmpq+64*1], m2
add tmpq, 64*2
sub hd, 2
jg .v_w32_loop
RET
.v_w64:
movu m0, [srcq+64*0]
movu m1, [srcq+64*1]
.v_w64_loop:
add srcq, strideq
pmullw m2, m8, m0
movu m0, [srcq+64*0]
pmullw m3, m8, m1
movu m1, [srcq+64*1]
pmullw m4, m9, m0
pmullw m5, m9, m1
psubw m2, m10
psubw m3, m10
paddw m2, m4
paddw m3, m5
psraw m2, 2
psraw m3, 2
mova [tmpq+64*0], m2
mova [tmpq+64*1], m3
add tmpq, 64*2
dec hd
jg .v_w64_loop
RET
.v_w128:
movu m0, [srcq+64*0]
movu m1, [srcq+64*1]
movu m2, [srcq+64*2]
movu m3, [srcq+64*3]
.v_w128_loop:
add srcq, strideq
pmullw m4, m8, m0
movu m0, [srcq+64*0]
pmullw m5, m8, m1
movu m1, [srcq+64*1]
pmullw m6, m8, m2
movu m2, [srcq+64*2]
pmullw m7, m8, m3
movu m3, [srcq+64*3]
pmullw m11, m9, m0
pmullw m12, m9, m1
pmullw m13, m9, m2
pmullw m14, m9, m3
REPX {psubw x, m10}, m4, m5, m6, m7
paddw m4, m11
paddw m5, m12
paddw m6, m13
paddw m7, m14
REPX {psraw x, 2}, m4, m5, m6, m7
mova [tmpq+64*0], m4
mova [tmpq+64*1], m5
mova [tmpq+64*2], m6
mova [tmpq+64*3], m7
add tmpq, 64*4
dec hd
jg .v_w128_loop
RET
.hv:
movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)]
shl mxyd, 11
vpbroadcastw m7, mxyd
add wq, r6
lea stride3q, [strideq*3]
jmp wq
.hv_w4:
movq xmm0, [srcq+strideq*0+0]
movq xmm1, [srcq+strideq*0+2]
pmullw xmm0, xm4
pmullw xmm1, xm5
psubw xmm0, xm6
paddw xmm0, xmm1
psraw xmm0, 2
vpbroadcastq ym0, xmm0
.hv_w4_loop:
movu xm1, [srcq+strideq*1]
vinserti128 ym1, [srcq+stride3q ], 1
movu xm2, [srcq+strideq*2]
lea srcq, [srcq+strideq*4]
vinserti128 ym2, [srcq+strideq*0], 1
punpcklqdq ym3, ym1, ym2
psrldq ym1, 2
psrldq ym2, 2
pmullw ym3, ym4
punpcklqdq ym1, ym2
pmullw ym1, ym5
psubw ym3, ym6
paddw ym1, ym3
psraw ym1, 2 ; 1 2 3 4
valignq ym2, ym1, ym0, 3 ; 0 1 2 3
mova ym0, ym1
psubw ym1, ym2
pmulhrsw ym1, ym7
paddw ym1, ym2
mova [tmpq], ym1
add tmpq, 32
sub hd, 4
jg .hv_w4_loop
RET
.hv_w8:
pmullw xm0, xm4, [srcq+strideq*0+0]
pmullw xm1, xm5, [srcq+strideq*0+2]
psubw xm0, xm6
paddw xm0, xm1
psraw xm0, 2
vinserti32x4 m0, xm0, 3
.hv_w8_loop:
movu xm1, [srcq+strideq*1+0]
movu xm2, [srcq+strideq*1+2]
vinserti32x4 ym1, [srcq+strideq*2+0], 1
vinserti32x4 ym2, [srcq+strideq*2+2], 1
vinserti32x4 m1, [srcq+stride3q +0], 2
vinserti32x4 m2, [srcq+stride3q +2], 2
lea srcq, [srcq+strideq*4]
vinserti32x4 m1, [srcq+strideq*0+0], 3
vinserti32x4 m2, [srcq+strideq*0+2], 3
pmullw m1, m4
pmullw m2, m5
psubw m1, m6
paddw m1, m2
psraw m1, 2 ; 1 2 3 4
valignq m2, m1, m0, 6 ; 0 1 2 3
mova m0, m1
psubw m1, m2
pmulhrsw m1, m7
paddw m1, m2
mova [tmpq], m1
add tmpq, 64
sub hd, 4
jg .hv_w8_loop
RET
.hv_w16:
pmullw ym0, ym4, [srcq+strideq*0+0]
pmullw ym1, ym5, [srcq+strideq*0+2]
psubw ym0, ym6
paddw ym0, ym1
psraw ym0, 2
vinserti32x8 m0, ym0, 1
.hv_w16_loop:
movu ym1, [srcq+strideq*1+0]
movu ym2, [srcq+strideq*1+2]
lea srcq, [srcq+strideq*2]
vinserti32x8 m1, [srcq+strideq*0+0], 1
vinserti32x8 m2, [srcq+strideq*0+2], 1
pmullw m1, m4
pmullw m2, m5
psubw m1, m6
paddw m1, m2
psraw m1, 2 ; 1 2
vshufi32x4 m2, m0, m1, q1032 ; 0 1
mova m0, m1
psubw m1, m2
pmulhrsw m1, m7
paddw m1, m2
mova [tmpq], m1
add tmpq, 64
sub hd, 2
jg .hv_w16_loop
RET
.hv_w32:
pmullw m0, m4, [srcq+strideq*0+0]
pmullw m1, m5, [srcq+strideq*0+2]
psubw m0, m6
paddw m0, m1
psraw m0, 2
.hv_w32_loop:
pmullw m3, m4, [srcq+strideq*1+0]
pmullw m1, m5, [srcq+strideq*1+2]
lea srcq, [srcq+strideq*2]
psubw m3, m6
paddw m3, m1
psraw m3, 2
psubw m1, m3, m0
pmulhrsw m1, m7
paddw m1, m0
pmullw m0, m4, [srcq+strideq*0+0]
pmullw m2, m5, [srcq+strideq*0+2]
psubw m0, m6
paddw m0, m2
psraw m0, 2
psubw m2, m0, m3
pmulhrsw m2, m7
paddw m2, m3
mova [tmpq+64*0], m1
mova [tmpq+64*1], m2
add tmpq, 64*2
sub hd, 2
jg .hv_w32_loop
RET
.hv_w64:
pmullw m0, m4, [srcq+ 0]
pmullw m2, m5, [srcq+ 2]
pmullw m1, m4, [srcq+64]
pmullw m3, m5, [srcq+66]
psubw m0, m6
psubw m1, m6
paddw m0, m2
paddw m1, m3
psraw m0, 2
psraw m1, 2
.hv_w64_loop:
add srcq, strideq
pmullw m2, m4, [srcq+ 0]
pmullw m8, m5, [srcq+ 2]
pmullw m3, m4, [srcq+64]
pmullw m9, m5, [srcq+66]
psubw m2, m6
psubw m3, m6
paddw m2, m8
paddw m3, m9
psraw m2, 2
psraw m3, 2
psubw m8, m2, m0
psubw m9, m3, m1
pmulhrsw m8, m7
pmulhrsw m9, m7
paddw m8, m0
mova m0, m2
paddw m9, m1
mova m1, m3
mova [tmpq+64*0], m8
mova [tmpq+64*1], m9
add tmpq, 64*2
dec hd
jg .hv_w64_loop
RET
.hv_w128:
pmullw m0, m4, [srcq+ 0]
pmullw m8, m5, [srcq+ 2]
pmullw m1, m4, [srcq+ 64]
pmullw m9, m5, [srcq+ 66]
pmullw m2, m4, [srcq+128]
pmullw m10, m5, [srcq+130]
pmullw m3, m4, [srcq+192]
pmullw m11, m5, [srcq+194]
REPX {psubw x, m6}, m0, m1, m2, m3
paddw m0, m8
paddw m1, m9
paddw m2, m10
paddw m3, m11
REPX {psraw x, 2}, m0, m1, m2, m3
.hv_w128_loop:
add srcq, strideq
pmullw m8, m4, [srcq+ 0]
pmullw m12, m5, [srcq+ 2]
pmullw m9, m4, [srcq+ 64]
pmullw m13, m5, [srcq+ 66]
pmullw m10, m4, [srcq+128]
pmullw m14, m5, [srcq+130]
pmullw m11, m4, [srcq+192]
pmullw m15, m5, [srcq+194]
REPX {psubw x, m6}, m8, m9, m10, m11
paddw m8, m12
paddw m9, m13
paddw m10, m14
paddw m11, m15
REPX {psraw x, 2}, m8, m9, m10, m11
psubw m12, m8, m0
psubw m13, m9, m1
psubw m14, m10, m2
psubw m15, m11, m3
REPX {pmulhrsw x, m7}, m12, m13, m14, m15
paddw m12, m0
mova m0, m8
paddw m13, m1
mova m1, m9
mova [tmpq+64*0], m12
mova [tmpq+64*1], m13
paddw m14, m2
mova m2, m10
paddw m15, m3
mova m3, m11
mova [tmpq+64*2], m14
mova [tmpq+64*3], m15
add tmpq, 64*4
dec hd
jg .hv_w128_loop
RET
; int8_t subpel_filters[5][15][8]
%assign FILTER_REGULAR (0*15 << 16) | 3*15
%assign FILTER_SMOOTH (1*15 << 16) | 4*15
%assign FILTER_SHARP (2*15 << 16) | 3*15
%macro MC_8TAP_FN 4 ; prefix, type, type_h, type_v
cglobal %1_8tap_%2_16bpc
mov t0d, FILTER_%3
%ifidn %3, %4
mov t1d, t0d
%else
mov t1d, FILTER_%4
%endif
%ifnidn %2, regular ; skip the jump in the last filter
jmp mangle(private_prefix %+ _%1_8tap_16bpc %+ SUFFIX)
%endif
%endmacro
%if WIN64
DECLARE_REG_TMP 4, 5
%define buf rsp+stack_offset+8 ; shadow space
%else
DECLARE_REG_TMP 7, 8
%define buf rsp-40 ; red zone
%endif
MC_8TAP_FN put, sharp, SHARP, SHARP
MC_8TAP_FN put, sharp_smooth, SHARP, SMOOTH
MC_8TAP_FN put, smooth_sharp, SMOOTH, SHARP
MC_8TAP_FN put, smooth, SMOOTH, SMOOTH
MC_8TAP_FN put, sharp_regular, SHARP, REGULAR
MC_8TAP_FN put, regular_sharp, REGULAR, SHARP
MC_8TAP_FN put, smooth_regular, SMOOTH, REGULAR
MC_8TAP_FN put, regular_smooth, REGULAR, SMOOTH
MC_8TAP_FN put, regular, REGULAR, REGULAR
cglobal put_8tap_16bpc, 4, 9, 16, dst, ds, src, ss, w, h, mx, my
%define base r8-put_avx512icl
imul mxd, mxm, 0x010101
add mxd, t0d ; 8tap_h, mx, 4tap_h
imul myd, mym, 0x010101
add myd, t1d ; 8tap_v, my, 4tap_v
lea r8, [put_avx512icl]
movifnidn wd, wm
movifnidn hd, hm
test mxd, 0xf00
jnz .h
test myd, 0xf00
jnz .v
tzcnt wd, wd
movzx wd, word [r8+wq*2+table_offset(put,)]
add wq, r8
%if WIN64
pop r8
%endif
jmp wq
.h_w2:
movzx mxd, mxb
sub srcq, 2
mova ym2, [spel_h_shuf2a]
pmovsxbw xmm4, [base+subpel_filters+mxq*8]
pshufd xmm3, xmm4, q1111
pshufd xmm4, xmm4, q2222
.h_w2_loop:
movu xm1, [srcq+ssq*0]
vinserti32x4 ym1, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
mova xmm0, xm8
vpermb ym1, ym2, ym1
vpdpwssd xmm0, xmm3, xm1
vextracti32x4 xm1, ym1, 1
vpdpwssd xmm0, xmm4, xm1
psrad xmm0, 6
packusdw xmm0, xmm0
pminsw xmm0, xm9
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w2_loop
RET
.h_w4:
movzx mxd, mxb
sub srcq, 2
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
vbroadcasti32x4 ym4, [spel_h_shufA]
vbroadcasti32x4 ym5, [spel_h_shufB]
pshufd xmm0, xmm0, q2211
vpbroadcastq ym6, xmm0
vpermq ym7, ymm0, q1111
.h_w4_loop:
movu xm2, [srcq+ssq*0]
vinserti32x4 ym2, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
mova ym0, ym8
pshufb ym1, ym2, ym4
vpdpwssd ym0, ym6, ym1
pshufb ym2, ym5
vpdpwssd ym0, ym7, ym2
psrad ym0, 6
vextracti32x4 xm1, ym0, 1
packusdw xm0, xm1
pminsw xmm0, xm0, xm9
movq [dstq+dsq*0], xmm0
movhps [dstq+dsq*1], xmm0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w4_loop
RET
.h:
test myd, 0xf00
jnz .hv
mov r7d, r8m
vpbroadcastw m9, r8m
shr r7d, 11
vpbroadcastd m8, [base+put_8tap_h_rnd+r7*4]
cmp wd, 4
je .h_w4
jl .h_w2
shr mxd, 16
sub srcq, 6
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
mova [buf], xmm0
vpbroadcastd m10, xmm0
vpbroadcastd m11, [buf+ 4]
vpbroadcastd m12, [buf+ 8]
vpbroadcastd m13, [buf+12]
sub wd, 16
je .h_w16
jg .h_w32
.h_w8:
mova m4, [spel_h_shufA]
movu m5, [spel_h_shufB]
movu m6, [spel_h_shufC]
mova m7, [spel_h_shufD]
.h_w8_loop:
movu ym2, [srcq+ssq*0]
vinserti32x8 m2, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
mova m0, m8
vpermb m1, m4, m2
vpdpwssd m0, m10, m1
vpermb m1, m5, m2
vpdpwssd m0, m11, m1
vpermb m1, m6, m2
vpdpwssd m0, m12, m1
vpermb m1, m7, m2
vpdpwssd m0, m13, m1
psrad m0, 6
vextracti32x8 ym1, m0, 1
packusdw ym0, ym1
pminsw ym0, ym9
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], ym0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w8_loop
RET
.h_w16:
vbroadcasti32x4 m6, [spel_h_shufA]
vbroadcasti32x4 m7, [spel_h_shufB]
.h_w16_loop:
movu ym2, [srcq+ssq*0+ 0]
vinserti32x8 m2, [srcq+ssq*1+ 0], 1
movu ym3, [srcq+ssq*0+16]
vinserti32x8 m3, [srcq+ssq*1+16], 1
lea srcq, [srcq+ssq*2]
mova m0, m8
mova m1, m8
pshufb m4, m2, m6
vpdpwssd m0, m10, m4 ; a0
pshufb m4, m3, m6
vpdpwssd m1, m12, m4 ; b2
pshufb m4, m2, m7
vpdpwssd m0, m11, m4 ; a1
pshufb m4, m3, m7
vpdpwssd m1, m13, m4 ; b3
shufpd m2, m3, 0x55
pshufb m4, m2, m6
vpdpwssd m0, m12, m4 ; a2
vpdpwssd m1, m10, m4 ; b0
pshufb m2, m7
vpdpwssd m0, m13, m2 ; a3
vpdpwssd m1, m11, m2 ; b1
psrad m0, 6
psrad m1, 6
packusdw m0, m1
pminsw m0, m9
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w16_loop
RET
.h_w32:
lea srcq, [srcq+wq*2]
vbroadcasti32x4 m6, [spel_h_shufA]
lea dstq, [dstq+wq*2]
vbroadcasti32x4 m7, [spel_h_shufB]
neg wq
.h_w32_loop0:
mov r6, wq
.h_w32_loop:
movu m2, [srcq+r6*2+ 0]
movu m3, [srcq+r6*2+ 8]
mova m0, m8
mova m1, m8
pshufb m4, m2, m6
vpdpwssd m0, m10, m4 ; a0
pshufb m4, m3, m6
vpdpwssd m1, m10, m4 ; b0
vpdpwssd m0, m12, m4 ; a2
movu m4, [srcq+r6*2+16]
pshufb m3, m7
vpdpwssd m1, m11, m3 ; b1
vpdpwssd m0, m13, m3 ; a3
pshufb m3, m4, m6
vpdpwssd m1, m12, m3 ; b2
pshufb m2, m7
vpdpwssd m0, m11, m2 ; a1
pshufb m4, m7
vpdpwssd m1, m13, m4 ; b3
psrad m0, 6
psrad m1, 6
packusdw m0, m1
pminsw m0, m9
mova [dstq+r6*2], m0
add r6, 32
jl .h_w32_loop
add srcq, ssq
add dstq, dsq
dec hd
jg .h_w32_loop0
RET
.v:
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
vpbroadcastd m10, [pd_32]
pmovsxbw xmm0, [base+subpel_filters+myq*8]
tzcnt r7d, wd
vpbroadcastw m11, r8m
lea r6, [ssq*3]
movzx r7d, word [r8+r7*2+table_offset(put, _8tap_v)]
sub srcq, r6
mova [rsp+stack_offset+8], xmm0
vpbroadcastd m12, xmm0
add r7, r8
vpbroadcastd m13, [rsp+stack_offset+12]
vpbroadcastd m14, [rsp+stack_offset+16]
vpbroadcastd m15, [rsp+stack_offset+20]
jmp r7
.v_w2:
movd xmm2, [srcq+ssq*0]
pinsrd xmm2, [srcq+ssq*1], 1
pinsrd xmm2, [srcq+ssq*2], 2
add srcq, r6
pinsrd xmm2, [srcq+ssq*0], 3 ; 0 1 2 3
movd xmm3, [srcq+ssq*1]
vpbroadcastd xmm1, [srcq+ssq*2]
add srcq, r6
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm3, xmm1, 0x02 ; 4 5
vpblendd xmm1, xmm0, 0x02 ; 5 6
palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4
punpcklwd xmm3, xmm1 ; 45 56
punpcklwd xmm1, xmm2, xmm4 ; 01 12
punpckhwd xmm2, xmm4 ; 23 34
.v_w2_loop:
vpbroadcastd xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova xmm5, xm10
vpdpwssd xmm5, xm12, xmm1 ; a0 b0
mova xmm1, xmm2
vpdpwssd xmm5, xm13, xmm2 ; a1 b1
mova xmm2, xmm3
vpdpwssd xmm5, xm14, xmm3 ; a2 b2
vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm4, xmm0, 0x02 ; 7 8
punpcklwd xmm3, xmm4 ; 67 78
vpdpwssd xmm5, xm15, xmm3 ; a3 b3
psrad xmm5, 6
packusdw xmm5, xmm5
pminsw xmm5, xm11
movd [dstq+dsq*0], xmm5
pextrd [dstq+dsq*1], xmm5, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w2_loop
RET
.v_w4:
movq xmm1, [srcq+ssq*0]
vpbroadcastq ymm0, [srcq+ssq*1]
vpbroadcastq ymm2, [srcq+ssq*2]
add srcq, r6
vpbroadcastq ymm4, [srcq+ssq*0]
vpbroadcastq ymm3, [srcq+ssq*1]
vpbroadcastq ymm5, [srcq+ssq*2]
add srcq, r6
vpblendd ymm1, ymm0, 0x30
vpblendd ymm0, ymm2, 0x30
punpcklwd ymm1, ymm0 ; 01 12
vpbroadcastq ymm0, [srcq+ssq*0]
vpblendd ymm2, ymm4, 0x30
vpblendd ymm4, ymm3, 0x30
punpcklwd ymm2, ymm4 ; 23 34
vpblendd ymm3, ymm5, 0x30
vpblendd ymm5, ymm0, 0x30
punpcklwd ymm3, ymm5 ; 45 56
.v_w4_loop:
vpbroadcastq ymm5, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova ymm4, ym10
vpdpwssd ymm4, ym12, ymm1 ; a0 b0
mova ymm1, ymm2
vpdpwssd ymm4, ym13, ymm2 ; a1 b1
mova ymm2, ymm3
vpdpwssd ymm4, ym14, ymm3 ; a2 b2
vpblendd ymm3, ymm0, ymm5, 0x30
vpbroadcastq ymm0, [srcq+ssq*0]
vpblendd ymm5, ymm0, 0x30
punpcklwd ymm3, ymm5 ; 67 78
vpdpwssd ymm4, ym15, ymm3 ; a3 b3
psrad ymm4, 6
vextracti128 xmm5, ymm4, 1
packusdw xmm4, xmm5
pminsw xmm4, xm11
movq [dstq+dsq*0], xmm4
movhps [dstq+dsq*1], xmm4
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w4_loop
vzeroupper
RET
.v_w8:
vbroadcasti32x4 m2, [srcq+ssq*2]
vinserti32x4 m1, m2, [srcq+ssq*0], 0
vinserti32x4 m1, [srcq+ssq*1], 1 ; 0 1 2
add srcq, r6
vinserti32x4 ym2, [srcq+ssq*0], 1
vinserti32x4 m2, [srcq+ssq*1], 2 ; 2 3 4
mova m6, [spel_v_shuf8]
movu xm0, [srcq+ssq*1]
vinserti32x4 ym0, [srcq+ssq*2], 1
add srcq, r6
vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6
vpermb m1, m6, m1 ; 01 12
vpermb m2, m6, m2 ; 23 34
vpermb m3, m6, m0 ; 45 56
.v_w8_loop:
vinserti32x4 m0, [srcq+ssq*1], 3
lea srcq, [srcq+ssq*2]
movu xm5, [srcq+ssq*0]
mova m4, m10
vpdpwssd m4, m12, m1 ; a0 b0
mova m1, m2
vshufi32x4 m0, m5, q1032 ; 6 7 8
vpdpwssd m4, m13, m2 ; a1 b1
mova m2, m3
vpdpwssd m4, m14, m3 ; a2 b2
vpermb m3, m6, m0 ; 67 78
vpdpwssd m4, m15, m3 ; a3 b3
psrad m4, 6
vextracti32x8 ym5, m4, 1
packusdw ym4, ym5
pminsw ym4, ym11
mova [dstq+dsq*0], xm4
vextracti32x4 [dstq+dsq*1], ym4, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w8_loop
RET
.v_w16:
vbroadcasti32x8 m1, [srcq+ssq*1]
vinserti32x8 m0, m1, [srcq+ssq*0], 0
vinserti32x8 m1, [srcq+ssq*2], 1
mova m8, [spel_v_shuf16]
add srcq, r6
movu ym3, [srcq+ssq*0]
vinserti32x8 m3, [srcq+ssq*1], 1
movu ym5, [srcq+ssq*2]
add srcq, r6
vinserti32x8 m5, [srcq+ssq*0], 1
vpermb m0, m8, m0 ; 01
vpermb m1, m8, m1 ; 12
vpermb m3, m8, m3 ; 34
vpermb m5, m8, m5 ; 56
mova m9, [deint_q_shuf]
vpshrdd m2, m1, m3, 16 ; 23
vpshrdd m4, m3, m5, 16 ; 45
.v_w16_loop:
mova m6, m10
mova m7, m10
vpdpwssd m6, m12, m0 ; a0
mova m0, m2
vpdpwssd m7, m12, m1 ; b0
mova m1, m3
vpdpwssd m6, m13, m2 ; a1
mova m2, m4
vpdpwssd m7, m13, m3 ; b1
mova m3, m5
vpdpwssd m6, m14, m4 ; a2
mova m4, m5
vpdpwssd m7, m14, m5 ; b2
movu ym5, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti32x8 m5, [srcq+ssq*0], 1
vpermb m5, m8, m5 ; 78
vpshrdd m4, m5, 16 ; 67
vpdpwssd m6, m15, m4 ; a3
vpdpwssd m7, m15, m5 ; b3
psrad m6, 6
psrad m7, 6
packusdw m6, m7
pminsw m6, m11
vpermq m6, m9, m6
mova [dstq+dsq*0], ym6
vextracti32x8 [dstq+dsq*1], m6, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w16_loop
RET
.v_w32:
.v_w64:
.v_w128:
%if WIN64
movaps [rsp+stack_offset+8], xmm6
%endif
lea wd, [hq+wq*8-256]
mov r7, srcq
mov r8, dstq
.v_w32_loop0:
movu m16, [srcq+ssq*0]
movu m17, [srcq+ssq*1]
movu m18, [srcq+ssq*2]
add srcq, r6
movu m19, [srcq+ssq*0]
movu m20, [srcq+ssq*1]
movu m21, [srcq+ssq*2]
add srcq, r6
movu m22, [srcq+ssq*0]
punpcklwd m0, m16, m17 ; 01l
punpckhwd m16, m17 ; 01h
punpcklwd m1, m17, m18 ; 12l
punpckhwd m17, m18 ; 12h
punpcklwd m2, m18, m19 ; 23l
punpckhwd m18, m19 ; 23h
punpcklwd m3, m19, m20 ; 34l
punpckhwd m19, m20 ; 34h
punpcklwd m4, m20, m21 ; 45l
punpckhwd m20, m21 ; 45h
punpcklwd m5, m21, m22 ; 56l
punpckhwd m21, m22 ; 56h
.v_w32_loop:
mova m6, m10
vpdpwssd m6, m12, m0 ; a0l
mova m8, m10
vpdpwssd m8, m12, m16 ; a0h
mova m7, m10
vpdpwssd m7, m12, m1 ; b0l
mova m9, m10
vpdpwssd m9, m12, m17 ; b0h
mova m0, m2
vpdpwssd m6, m13, m2 ; a1l
mova m16, m18
vpdpwssd m8, m13, m18 ; a1h
mova m1, m3
vpdpwssd m7, m13, m3 ; b1l
mova m17, m19
vpdpwssd m9, m13, m19 ; b1h
mova m2, m4
vpdpwssd m6, m14, m4 ; a2l
mova m18, m20
vpdpwssd m8, m14, m20 ; a2h
mova m3, m5
vpdpwssd m7, m14, m5 ; b2l
mova m19, m21
vpdpwssd m9, m14, m21 ; b2h
movu m21, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
punpcklwd m4, m22, m21 ; 67l
punpckhwd m20, m22, m21 ; 67h
movu m22, [srcq+ssq*0]
vpdpwssd m6, m15, m4 ; a3l
vpdpwssd m8, m15, m20 ; a3h
punpcklwd m5, m21, m22 ; 78l
punpckhwd m21, m22 ; 78h
vpdpwssd m7, m15, m5 ; b3l
vpdpwssd m9, m15, m21 ; b3h
REPX {psrad x, 6}, m6, m8, m7, m9
packusdw m6, m8
packusdw m7, m9
pminsw m6, m11
pminsw m7, m11
mova [dstq+dsq*0], m6
mova [dstq+dsq*1], m7
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w32_loop
add r7, 64
add r8, 64
movzx hd, wb
mov srcq, r7
mov dstq, r8
sub wd, 1<<8
jg .v_w32_loop0
%if WIN64
movaps xmm6, [rsp+stack_offset+8]
%endif
vzeroupper
RET
.hv:
vpbroadcastw m11, r8m
cmp wd, 4
jg .hv_w8
movzx mxd, mxb
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
pmovsxbw xmm1, [base+subpel_filters+myq*8]
lea r6, [ssq*3]
sub srcq, 2
sub srcq, r6
test dword r8m, 0x800
jnz .hv_12bit
vpbroadcastd m10, [pd_2176]
psllw xmm0, 6
jmp .hv_main
.hv_12bit:
vpbroadcastd m10, [pd_640]
psllw xmm0, 4
psllw xmm1, 2
.hv_main:
mova [buf+ 0], xmm0
mova [buf+16], xmm1
vpbroadcastd m8, [buf+ 4]
vpbroadcastd m9, [buf+ 8]
vpbroadcastd ym12, xmm1
vpbroadcastd ym13, [buf+20]
vpbroadcastd ym14, [buf+24]
vpbroadcastd ym15, [buf+28]
movu xm4, [srcq+ssq*0]
vinserti32x4 ym4, [srcq+ssq*1], 1
vinserti32x4 m4, [srcq+ssq*2], 2
add srcq, r6
vinserti32x4 m4, [srcq+ssq*0], 3 ; 0 1 2 3
movu xm0, [srcq+ssq*1]
vinserti32x4 ym0, [srcq+ssq*2], 1
add srcq, r6
vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6
cmp wd, 4
je .hv_w4
vbroadcasti32x4 m2, [spel_h_shufA]
mova m3, [spel_h_shuf2b]
mova ym6, [spel_h_shuf2a]
mova xm7, [spel_shuf2]
mova m1, m10
pshufb m4, m2
pshufb m0, m2
punpcklqdq m2, m4, m0
vpdpwssd m1, m8, m2 ; 04 15 26 3_
punpckhqdq m4, m0
vpdpwssd m1, m9, m4
vpermb m1, m3, m1 ; 01 12
vextracti32x4 xm2, ym1, 1 ; 23 34
vextracti32x4 xm3, m1, 2 ; 45 56
.hv_w2_loop:
movu xm5, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti32x4 ym5, [srcq+ssq*0], 1
mova xm4, xm10
vpermb ym5, ym6, ym5
pmaddwd xmm0, xm12, xm1 ; a0 b0
vpdpwssd xm4, xm8, xm5
vextracti32x4 xm5, ym5, 1
mova xm1, xm2
vpdpwssd xmm0, xm13, xm2 ; a1 b1
vpdpwssd xm4, xm9, xm5 ; 7 8
mova xm2, xm3
vpdpwssd xmm0, xm14, xm3 ; a2 b2
vpermt2b xm3, xm7, xm4 ; 67 78
vpdpwssd xmm0, xm15, xm3 ; a3 b3
psrad xmm0, 10
packusdw xmm0, xmm0
pminsw xmm0, xm11
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w2_loop
RET
.hv_w4:
vbroadcasti32x4 m19, [spel_h_shufA]
vbroadcasti32x4 m20, [spel_h_shufB]
mova ym6, [spel_shuf4a]
mova ym7, [spel_shuf4b]
mova m2, m10
mova m3, m10
pshufb m1, m4, m19
vpdpwssd m2, m8, m1
pshufb m1, m0, m19
vpdpwssd m3, m8, m1
pshufb m4, m20
vpdpwssd m2, m9, m4
pshufb m0, m20
vpdpwssd m3, m9, m0
vpermb m1, m6, m2 ; 01 12
vshufi32x4 m2, m3, q1032
vpermb m3, m6, m3 ; 45 56
vpermb m2, m6, m2 ; 23 34
.hv_w4_loop:
movu xm18, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti128 ym18, [srcq+ssq*0], 1
mova ym4, ym10
pshufb ym17, ym18, ym19
pmaddwd ym16, ym12, ym1 ; a0 b0
vpdpwssd ym4, ym8, ym17
pshufb ym18, ym20
mova ym1, ym2
vpdpwssd ym16, ym13, ym2 ; a1 b1
vpdpwssd ym4, ym9, ym18 ; 7 8
mova ym2, ym3
vpdpwssd ym16, ym14, ym3 ; a2 b2
vpermt2b ym3, ym7, ym4 ; 67 78
vpdpwssd ym16, ym15, ym3 ; a3 b3
psrad ym16, 10
vextracti128 xm17, ym16, 1
packusdw xm16, xm17
pminsw xm16, xm11
movq [dstq+dsq*0], xm16
movhps [dstq+dsq*1], xm16
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w4_loop
vzeroupper
RET
.hv_w8:
shr mxd, 16
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
pmovsxbw xmm1, [base+subpel_filters+myq*8]
lea r6, [ssq*3]
sub srcq, 6
sub srcq, r6
test dword r8m, 0x800
jnz .hv_w8_12bit
vpbroadcastd m10, [pd_2176]
psllw xmm0, 6
jmp .hv_w8_main
.hv_w8_12bit:
vpbroadcastd m10, [pd_640]
psllw xmm0, 4
psllw xmm1, 2
.hv_w8_main:
mova [buf+ 0], xmm0
mova [buf+16], xmm1
vpbroadcastd m12, xmm0
vpbroadcastd m13, [buf+ 4]
vpbroadcastd m14, [buf+ 8]
vpbroadcastd m15, [buf+12]
vpbroadcastd m16, xmm1
vpbroadcastd m17, [buf+20]
vpbroadcastd m18, [buf+24]
vpbroadcastd m19, [buf+28]
cmp wd, 16
je .hv_w16
jg .hv_w32
mova m5, [spel_h_shufA]
movu ym0, [srcq+ssq*0]
vinserti32x8 m0, [srcq+ssq*1], 1 ; 0 1
movu ym9, [srcq+ssq*2]
add srcq, r6
vinserti32x8 m9, [srcq+ssq*0], 1 ; 2 3
movu ym20, [srcq+ssq*1]
vinserti32x8 m20, [srcq+ssq*2], 1 ; 4 5
add srcq, r6
movu ym21, [srcq+ssq*0] ; 6
movu m6, [spel_h_shufB]
movu m7, [spel_h_shufC]
vpermb m8, m5, m0
mova m1, m10
vpdpwssd m1, m12, m8 ; a0 b0
vpermb m8, m5, m9
mova m2, m10
vpdpwssd m2, m12, m8 ; c0 d0
vpermb m8, m5, m20
mova m3, m10
vpdpwssd m3, m12, m8 ; e0 f0
vpermb m8, m5, m21
mova m4, m10
vpdpwssd m4, m12, m8 ; g0
vpermb m8, m6, m0
vpdpwssd m1, m13, m8 ; a1 b1
vpermb m8, m6, m9
vpdpwssd m2, m13, m8 ; c1 d1
vpermb m8, m6, m20
vpdpwssd m3, m13, m8 ; e1 f1
vpermb m8, m6, m21
vpdpwssd m4, m13, m8 ; g1
vpermb m8, m7, m0
vpdpwssd m1, m14, m8 ; a2 b2
vpermb m8, m7, m9
vpdpwssd m2, m14, m8 ; c2 d2
vpermb m8, m7, m20
vpdpwssd m3, m14, m8 ; e2 f2
vpermb m8, m7, m21
vpdpwssd m4, m14, m8 ; g2
mova m8, [spel_h_shufD]
vpermb m0, m8, m0
vpdpwssd m1, m15, m0 ; a3 b3
mova m0, [spel_shuf8a]
vpermb m9, m8, m9
vpdpwssd m2, m15, m9 ; c3 d3
mova m9, [spel_shuf8b]
vpermb m20, m8, m20
vpdpwssd m3, m15, m20 ; e3 f3
vpermb m21, m8, m21
vpdpwssd m4, m15, m21 ; g3
vpermt2b m1, m0, m2 ; 01 12
vpermt2b m2, m0, m3 ; 23 34
vpermt2b m3, m0, m4 ; 45 56
.hv_w8_loop:
movu ym0, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti32x8 m0, [srcq+ssq*0], 1
mova m4, m10
vpermb m21, m5, m0
vpdpwssd m4, m12, m21 ; h0 i0
vpermb m21, m6, m0
pmaddwd m20, m16, m1 ; A0 B0
vpdpwssd m4, m13, m21 ; h1 i1
vpermb m21, m7, m0
mova m1, m2
vpdpwssd m20, m17, m2 ; A1 B1
vpdpwssd m4, m14, m21 ; h2 i2
vpermb m21, m8, m0
mova m2, m3
vpdpwssd m20, m18, m3 ; A2 B2
vpdpwssd m4, m15, m21 ; h3 i3
vpermt2b m3, m9, m4 ; 67 78
vpdpwssd m20, m19, m3 ; A3 B3
psrad m20, 10
vextracti32x8 ym21, m20, 1
packusdw ym20, ym21
pminsw ym20, ym11
mova [dstq+dsq*0], xm20
vextracti128 [dstq+dsq*1], ym20, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w8_loop
vzeroupper
RET
.hv_w16:
WIN64_SPILL_XMM 26
vbroadcasti32x8 m5, [srcq+ssq*0+ 8]
vinserti32x8 m4, m5, [srcq+ssq*0+ 0], 0
vinserti32x8 m5, [srcq+ssq*0+16], 1 ; 0
movu ym6, [srcq+ssq*1+ 0]
movu ym7, [srcq+ssq*1+16]
vinserti32x8 m6, [srcq+ssq*2+ 0], 1
vinserti32x8 m7, [srcq+ssq*2+16], 1 ; 1 2
add srcq, r6
movu ym22, [srcq+ssq*0+ 0]
movu ym23, [srcq+ssq*0+16]
vinserti32x8 m22, [srcq+ssq*1+ 0], 1
vinserti32x8 m23, [srcq+ssq*1+16], 1 ; 3 4
movu ym24, [srcq+ssq*2+ 0]
movu ym25, [srcq+ssq*2+16]
add srcq, r6
vinserti32x8 m24, [srcq+ssq*0+ 0], 1
vinserti32x8 m25, [srcq+ssq*0+16], 1 ; 5 6
vbroadcasti32x4 m20, [spel_h_shufA]
vbroadcasti32x4 m21, [spel_h_shufB]
mova m9, [spel_shuf16]
pshufb m0, m4, m20
mova m1, m10
vpdpwssd m1, m12, m0 ; a0
pshufb m0, m6, m20
mova m2, m10
vpdpwssd m2, m12, m0 ; b0
pshufb m0, m7, m20
mova m3, m10
vpdpwssd m3, m14, m0 ; c2
pshufb m0, m4, m21
vpdpwssd m1, m13, m0 ; a1
pshufb m0, m6, m21
vpdpwssd m2, m13, m0 ; b1
pshufb m0, m7, m21
vpdpwssd m3, m15, m0 ; c3
pshufb m0, m5, m20
vpdpwssd m1, m14, m0 ; a2
shufpd m6, m7, 0x55
pshufb m7, m6, m20
vpdpwssd m2, m14, m7 ; b2
vpdpwssd m3, m12, m7 ; c0
pshufb m5, m21
vpdpwssd m1, m15, m5 ; a3
pshufb m6, m21
vpdpwssd m2, m15, m6 ; b3
vpdpwssd m3, m13, m6 ; c1
pshufb m0, m22, m20
mova m4, m10
vpdpwssd m4, m12, m0 ; d0
pshufb m0, m23, m20
mova m5, m10
vpdpwssd m5, m14, m0 ; e2
pshufb m0, m24, m20
mova m6, m10
vpdpwssd m6, m12, m0 ; f0
pshufb m0, m25, m20
mova m7, m10
vpdpwssd m7, m14, m0 ; g2
pshufb m0, m22, m21
vpdpwssd m4, m13, m0 ; d1
pshufb m0, m23, m21
vpdpwssd m5, m15, m0 ; e3
pshufb m0, m24, m21
vpdpwssd m6, m13, m0 ; f1
pshufb m0, m25, m21
vpdpwssd m7, m15, m0 ; g3
shufpd m22, m23, 0x55
pshufb m23, m22, m20
vpdpwssd m4, m14, m23 ; d2
vpdpwssd m5, m12, m23 ; e0
shufpd m24, m25, 0x55
pshufb m25, m24, m20
vpdpwssd m6, m14, m25 ; f2
vpdpwssd m7, m12, m25 ; g0
pshufb m22, m21
vpdpwssd m4, m15, m22 ; d3
vpdpwssd m5, m13, m22 ; e1
pshufb m24, m21
vpdpwssd m6, m15, m24 ; f3
vpdpwssd m7, m13, m24 ; g1
pslldq m1, 1
vpermt2b m2, m9, m3 ; 12
vpermt2b m4, m9, m5 ; 34
vpermt2b m6, m9, m7 ; 56
vpshrdd m1, m2, 16 ; 01
vpshrdd m3, m2, m4, 16 ; 23
vpshrdd m5, m4, m6, 16 ; 45
.hv_w16_loop:
movu ym24, [srcq+ssq*1+ 0]
movu ym25, [srcq+ssq*1+16]
lea srcq, [srcq+ssq*2]
vinserti32x8 m24, [srcq+ssq*0+ 0], 1
vinserti32x8 m25, [srcq+ssq*0+16], 1
mova m7, m10
mova m8, m10
pshufb m0, m24, m20
vpdpwssd m7, m12, m0 ; h0
pshufb m0, m25, m20
vpdpwssd m8, m14, m0 ; i2
pmaddwd m22, m16, m1 ; A0
mova m1, m3
pmaddwd m23, m16, m2 ; B0
mova m2, m4
pshufb m0, m24, m21
vpdpwssd m7, m13, m0 ; h1
pshufb m0, m25, m21
vpdpwssd m8, m15, m0 ; i3
vpdpwssd m22, m17, m3 ; A1
mova m3, m5
vpdpwssd m23, m17, m4 ; B1
mova m4, m6
shufpd m24, m25, 0x55
pshufb m25, m24, m20
vpdpwssd m7, m14, m25 ; h2
vpdpwssd m8, m12, m25 ; i0
vpdpwssd m22, m18, m5 ; A2
vpdpwssd m23, m18, m6 ; B2
pshufb m24, m21
vpdpwssd m7, m15, m24 ; h3
vpdpwssd m8, m13, m24 ; i1
vpermt2b m7, m9, m8 ; 78
vpshrdd m5, m6, m7, 16 ; 67
vpdpwssd m22, m19, m5 ; A3
vpdpwssd m23, m19, m7 ; B3
mova m6, m7
psrad m22, 10
psrad m23, 10
vshufi32x4 m0, m22, m23, q3232
vinserti32x8 m22, ym23, 1
packusdw m22, m0
pminsw m22, m11
mova [dstq+dsq*0], ym22
vextracti32x8 [dstq+dsq*1], m22, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w16_loop
RET
.hv_w32:
WIN64_SPILL_XMM 32
vbroadcasti32x4 m20, [spel_h_shufA]
vbroadcasti32x4 m21, [spel_h_shufB]
mova m22, [spel_shuf32]
lea wd, [hq+wq*8-256]
mov r7, srcq
mov r8, dstq
.hv_w32_loop0:
movu m6, [srcq+ssq*0+ 0]
movu m7, [srcq+ssq*0+ 8]
movu m8, [srcq+ssq*0+16]
mova m0, m10
mova m23, m10
pshufb m9, m6, m20
vpdpwssd m0, m12, m9 ; a0l
pshufb m9, m7, m20
vpdpwssd m23, m12, m9 ; a0h
vpdpwssd m0, m14, m9 ; a2l
pshufb m7, m21
vpdpwssd m23, m13, m7 ; a1h
vpdpwssd m0, m15, m7 ; a3l
pshufb m7, m8, m20
vpdpwssd m23, m14, m7 ; a2h
pshufb m6, m21
vpdpwssd m0, m13, m6 ; a1l
pshufb m8, m21
vpdpwssd m23, m15, m8 ; a3h
%macro PUT_8TAP_HV_W32 5 ; dst_lo, dst_hi, stride_name, stride[1-2]
movu m6, [srcq+%3*%4+ 0]
movu m7, [srcq+%3*%4+ 8]
movu m8, [srcq+%3*%4+16]
%if %4 == 2
add srcq, r6
%endif
movu m29, [srcq+%3*%5+ 0]
movu m30, [srcq+%3*%5+ 8]
movu m31, [srcq+%3*%5+16]
%if %5 == 2
add srcq, r6
%endif
mova m%1, m10
mova m9, m10
pshufb m%2, m6, m20
vpdpwssd m%1, m12, m%2 ; x0l
pshufb m%2, m29, m20
vpdpwssd m9, m12, m%2 ; y0l
pshufb m6, m21
vpdpwssd m%1, m13, m6 ; x1l
pshufb m29, m21
vpdpwssd m9, m13, m29 ; y1l
pshufb m6, m7, m20
mova m%2, m10
vpdpwssd m%2, m12, m6 ; x0h
pshufb m29, m30, m20
vpdpwssd m%1, m14, m6 ; y2l
mova m6, m10
vpdpwssd m6, m12, m29 ; x0h
pshufb m7, m21
vpdpwssd m9, m14, m29 ; y2l
pshufb m30, m21
vpdpwssd m%2, m13, m7 ; x1h
vpdpwssd m%1, m15, m7 ; x3l
pshufb m7, m8, m20
vpdpwssd m6, m13, m30 ; y1h
vpdpwssd m9, m15, m30 ; y3l
pshufb m30, m31, m20
vpdpwssd m%2, m14, m7 ; x2h
pshufb m8, m21
vpdpwssd m6, m14, m30 ; y2h
pshufb m31, m21
vpdpwssd m%2, m15, m8 ; x3h
vpdpwssd m6, m15, m31 ; y3h
%if %1 == 1
vpermt2b m0, m22, m%1 ; 01l
vpermt2b m23, m22, m%2 ; 01h
%endif
vpermt2b m%1, m22, m9 ; xyl
vpermt2b m%2, m22, m6 ; xyh
%endmacro
PUT_8TAP_HV_W32 1, 24, ssq, 1, 2 ; 12
PUT_8TAP_HV_W32 3, 26, ssq, 0, 1 ; 34
PUT_8TAP_HV_W32 5, 28, ssq, 2, 0 ; 56
vpshrdd m2, m1, m3, 16 ; 23l
vpshrdd m25, m24, m26, 16 ; 23h
vpshrdd m4, m3, m5, 16 ; 45l
vpshrdd m27, m26, m28, 16 ; 45h
.hv_w32_loop:
movu m7, [srcq+ssq*1+ 0]
movu m9, [srcq+ssq*2+ 0]
movu m6, [srcq+ssq*1+ 8]
movu m8, [srcq+ssq*2+ 8]
mova m29, m10
mova m31, m10
pshufb m30, m7, m20
vpdpwssd m29, m12, m30 ; h0l
pshufb m30, m9, m20
vpdpwssd m31, m12, m30 ; i0l
pshufb m7, m21
vpdpwssd m29, m13, m7 ; h1l
pshufb m9, m21
vpdpwssd m31, m13, m9 ; i1l
pshufb m7, m6, m20
vpdpwssd m29, m14, m7 ; h2l
pshufb m9, m8, m20
vpdpwssd m31, m14, m9 ; i2l
pshufb m6, m21
vpdpwssd m29, m15, m6 ; h3l
pshufb m8, m21
vpdpwssd m31, m15, m8 ; i3l
mova m30, m10
vpdpwssd m30, m12, m7 ; h0h
movu m7, [srcq+ssq*1+16]
lea srcq, [srcq+ssq*2]
vpermt2b m29, m22, m31 ; 78l
mova m31, m10
vpdpwssd m31, m12, m9 ; i0h
movu m9, [srcq+ssq*0+16]
vpdpwssd m30, m13, m6 ; h1h
pshufb m6, m7, m20
vpdpwssd m31, m13, m8 ; i1h
pshufb m8, m9, m20
vpdpwssd m30, m14, m6 ; h2h
pmaddwd m6, m16, m0 ; A0l
pshufb m7, m21
vpdpwssd m31, m14, m8 ; i2h
pmaddwd m8, m16, m23 ; A0h
pshufb m9, m21
vpdpwssd m30, m15, m7 ; h3h
pmaddwd m7, m16, m1 ; B0l
vpdpwssd m31, m15, m9 ; i3h
pmaddwd m9, m16, m24 ; B0h
mova m0, m2
vpdpwssd m6, m17, m2 ; A1l
mova m23, m25
vpdpwssd m8, m17, m25 ; A1h
mova m1, m3
vpdpwssd m7, m17, m3 ; B1l
mova m24, m26
vpdpwssd m9, m17, m26 ; B1h
vpermt2b m30, m22, m31 ; 78h
vpdpwssd m6, m18, m4 ; A2l
mova m2, m4
vpdpwssd m8, m18, m27 ; A2h
mova m25, m27
vpdpwssd m7, m18, m5 ; B2l
mova m3, m5
vpdpwssd m9, m18, m28 ; B2h
mova m26, m28
vpshrdd m4, m5, m29, 16 ; 67l
vpdpwssd m6, m19, m4 ; A3l
vpshrdd m27, m28, m30, 16 ; 67h
vpdpwssd m8, m19, m27 ; A3h
mova m5, m29
vpdpwssd m7, m19, m29 ; B3l
mova m28, m30
vpdpwssd m9, m19, m30 ; B3h
REPX {psrad x, 10}, m6, m8, m7, m9
packusdw m6, m8
packusdw m7, m9
pminsw m6, m11
pminsw m7, m11
mova [dstq+dsq*0], m6
mova [dstq+dsq*1], m7
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w32_loop
add r7, 64
add r8, 64
movzx hd, wb
mov srcq, r7
mov dstq, r8
sub wd, 1<<8
jg .hv_w32_loop0
RET
%if WIN64
DECLARE_REG_TMP 6, 4
%else
DECLARE_REG_TMP 6, 7
%endif
MC_8TAP_FN prep, sharp, SHARP, SHARP
MC_8TAP_FN prep, sharp_smooth, SHARP, SMOOTH
MC_8TAP_FN prep, smooth_sharp, SMOOTH, SHARP
MC_8TAP_FN prep, smooth, SMOOTH, SMOOTH
MC_8TAP_FN prep, sharp_regular, SHARP, REGULAR
MC_8TAP_FN prep, regular_sharp, REGULAR, SHARP
MC_8TAP_FN prep, smooth_regular, SMOOTH, REGULAR
MC_8TAP_FN prep, regular_smooth, REGULAR, SMOOTH
MC_8TAP_FN prep, regular, REGULAR, REGULAR
cglobal prep_8tap_16bpc, 3, 8, 16, tmp, src, stride, w, h, mx, my, stride3
%define base r7-prep_avx512icl
imul mxd, mxm, 0x010101
add mxd, t0d ; 8tap_h, mx, 4tap_h
imul myd, mym, 0x010101
add myd, t1d ; 8tap_v, my, 4tap_v
lea r7, [prep_avx512icl]
mov wd, wm
movifnidn hd, hm
test mxd, 0xf00
jnz .h
test myd, 0xf00
jnz .v
tzcnt wd, wd
mov r5d, r7m ; bitdepth_max
vpbroadcastd m5, [pw_8192]
movzx wd, word [r7+wq*2+table_offset(prep,)]
shr r5d, 11
vpbroadcastd m4, [r7-prep_avx512icl+prep_mul+r5*4]
add wq, r7
lea r6, [strideq*3]
%if WIN64
pop r7
%endif
jmp wq
.h_w4:
movzx mxd, mxb
sub srcq, 2
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
mov r5d, r7m
vbroadcasti32x4 m4, [spel_h_shufA]
vbroadcasti32x4 m5, [spel_h_shufB]
shr r5d, 11
mova ym9, [prep_endA]
psllw xmm0, [base+prep_hv_shift+r5*8]
mova [tmpq], xmm0
vpbroadcastd m6, [tmpq+4]
vpbroadcastd m7, [tmpq+8]
.h_w4_loop:
movu xm2, [srcq+strideq*0]
vinserti32x4 ym2, [srcq+strideq*1], 1
vinserti32x4 m2, [srcq+strideq*2], 2
vinserti32x4 m2, [srcq+r6 ], 3
lea srcq, [srcq+strideq*4]
mova m0, m10
pshufb m1, m2, m4
vpdpwssd m0, m6, m1
pshufb m2, m5
vpdpwssd m0, m7, m2
vpermb m0, m9, m0
mova [tmpq], ym0
add tmpq, 32
sub hd, 4
jg .h_w4_loop
RET
.h:
test myd, 0xf00
jnz .hv
vpbroadcastd m10, [prep_8tap_rnd]
lea r6, [strideq*3]
cmp wd, 4
je .h_w4
shr mxd, 16
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
mov r5d, r7m
sub srcq, 6
shr r5d, 11
psllw xmm0, [base+prep_hv_shift+r5*8]
mova [tmpq], xmm0
vpbroadcastd m12, xmm0
vpbroadcastd m13, [tmpq+ 4]
vpbroadcastd m14, [tmpq+ 8]
vpbroadcastd m15, [tmpq+12]
cmp wd, 16
je .h_w16
jg .h_w32
.h_w8:
mova m6, [spel_h_shufA]
movu m7, [spel_h_shufB]
movu m8, [spel_h_shufC]
mova m9, [spel_h_shufD]
mova m11, [prep_endB]
.h_w8_loop:
movu ym4, [srcq+strideq*0]
vinserti32x8 m4, [srcq+strideq*1], 1
movu ym5, [srcq+strideq*2]
vinserti32x8 m5, [srcq+r6 ], 1
lea srcq, [srcq+strideq*4]
mova m0, m10
mova m1, m10
vpermb m2, m6, m4
vpermb m3, m6, m5
vpdpwssd m0, m12, m2
vpdpwssd m1, m12, m3
vpermb m2, m7, m4
vpermb m3, m7, m5
vpdpwssd m0, m13, m2
vpdpwssd m1, m13, m3
vpermb m2, m8, m4
vpermb m3, m8, m5
vpdpwssd m0, m14, m2
vpdpwssd m1, m14, m3
vpermb m2, m9, m4
vpermb m3, m9, m5
vpdpwssd m0, m15, m2
vpdpwssd m1, m15, m3
vpermt2b m0, m11, m1
mova [tmpq], m0
add tmpq, 64
sub hd, 4
jg .h_w8_loop
RET
.h_w16:
vbroadcasti32x4 m6, [spel_h_shufA]
vbroadcasti32x4 m7, [spel_h_shufB]
mova m11, [prep_endC]
.h_w16_loop:
movu ym2, [srcq+strideq*0+ 0]
vinserti32x8 m2, [srcq+strideq*1+ 0], 1
movu ym3, [srcq+strideq*0+16]
vinserti32x8 m3, [srcq+strideq*1+16], 1
lea srcq, [srcq+strideq*2]
mova m0, m10
mova m1, m10
pshufb m4, m2, m6
vpdpwssd m0, m12, m4 ; a0
pshufb m4, m3, m6
vpdpwssd m1, m14, m4 ; b2
pshufb m4, m2, m7
vpdpwssd m0, m13, m4 ; a1
pshufb m4, m3, m7
vpdpwssd m1, m15, m4 ; b3
shufpd m2, m3, 0x55
pshufb m4, m2, m6
vpdpwssd m0, m14, m4 ; a2
vpdpwssd m1, m12, m4 ; b0
pshufb m2, m7
vpdpwssd m0, m15, m2 ; a3
vpdpwssd m1, m13, m2 ; b1
vpermt2b m0, m11, m1
mova [tmpq], m0
add tmpq, 64
sub hd, 2
jg .h_w16_loop
RET
.h_w32:
vbroadcasti32x4 m6, [spel_h_shufA]
lea srcq, [srcq+wq*2]
vbroadcasti32x4 m7, [spel_h_shufB]
neg wq
mova m11, [prep_endC]
.h_w32_loop0:
mov r6, wq
.h_w32_loop:
movu m2, [srcq+r6*2+ 0]
movu m3, [srcq+r6*2+ 8]
mova m0, m10
mova m1, m10
pshufb m4, m2, m6
vpdpwssd m0, m12, m4 ; a0
pshufb m4, m3, m6
vpdpwssd m1, m12, m4 ; b0
vpdpwssd m0, m14, m4 ; a2
movu m4, [srcq+r6*2+16]
pshufb m3, m7
vpdpwssd m1, m13, m3 ; b1
vpdpwssd m0, m15, m3 ; a3
pshufb m3, m4, m6
vpdpwssd m1, m14, m3 ; b2
pshufb m2, m7
vpdpwssd m0, m13, m2 ; a1
pshufb m4, m7
vpdpwssd m1, m15, m4 ; b3
vpermt2b m0, m11, m1
mova [tmpq], m0
add tmpq, 64
add r6, 32
jl .h_w32_loop
add srcq, strideq
dec hd
jg .h_w32_loop0
RET
.v:
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
mov r5d, r7m
vpbroadcastd m10, [prep_8tap_rnd]
pmovsxbw xmm0, [base+subpel_filters+myq*8]
tzcnt r6d, wd
shr r5d, 11
movzx r6d, word [r7+r6*2+table_offset(prep, _8tap_v)]
psllw xmm0, [base+prep_hv_shift+r5*8]
add r7, r6
lea r6, [strideq*3]
sub srcq, r6
mova [tmpq], xmm0
vpbroadcastd m12, xmm0
vpbroadcastd m13, [tmpq+ 4]
vpbroadcastd m14, [tmpq+ 8]
vpbroadcastd m15, [tmpq+12]
jmp r7
.v_w4:
movq xmm1, [srcq+strideq*0]
vpbroadcastq ymm0, [srcq+strideq*1]
vpbroadcastq ymm2, [srcq+strideq*2]
add srcq, r6
vpbroadcastq ymm4, [srcq+strideq*0]
vpbroadcastq ymm3, [srcq+strideq*1]
vpbroadcastq ymm5, [srcq+strideq*2]
mova xm11, [prep_endA]
add srcq, r6
vpblendd ymm1, ymm0, 0x30
vpblendd ymm0, ymm2, 0x30
punpcklwd ymm1, ymm0 ; 01 12
vpbroadcastq ymm0, [srcq+strideq*0]
vpblendd ymm2, ymm4, 0x30
vpblendd ymm4, ymm3, 0x30
punpcklwd ymm2, ymm4 ; 23 34
vpblendd ymm3, ymm5, 0x30
vpblendd ymm5, ymm0, 0x30
punpcklwd ymm3, ymm5 ; 45 56
.v_w4_loop:
vpbroadcastq ymm5, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
mova ymm4, ym10
vpdpwssd ymm4, ym12, ymm1 ; a0 b0
mova ymm1, ymm2
vpdpwssd ymm4, ym13, ymm2 ; a1 b1
mova ymm2, ymm3
vpdpwssd ymm4, ym14, ymm3 ; a2 b2
vpblendd ymm3, ymm0, ymm5, 0x30
vpbroadcastq ymm0, [srcq+strideq*0]
vpblendd ymm5, ymm0, 0x30
punpcklwd ymm3, ymm5 ; 67 78
vpdpwssd ymm4, ym15, ymm3 ; a3 b3
vpermb ymm4, ym11, ymm4
mova [tmpq], xmm4
add tmpq, 16
sub hd, 2
jg .v_w4_loop
vzeroupper
RET
.v_w8:
vbroadcasti32x4 m2, [srcq+strideq*2]
vinserti32x4 m1, m2, [srcq+strideq*0], 0
vinserti32x4 m1, [srcq+strideq*1], 1 ; 0 1 2
add srcq, r6
vinserti32x4 ym2, [srcq+strideq*0], 1
vinserti32x4 m2, [srcq+strideq*1], 2 ; 2 3 4
mova m6, [spel_v_shuf8]
movu xm0, [srcq+strideq*1]
vinserti32x4 ym0, [srcq+strideq*2], 1
add srcq, r6
vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6
mova ym11, [prep_endB]
vpermb m1, m6, m1 ; 01 12
vpermb m2, m6, m2 ; 23 34
vpermb m3, m6, m0 ; 45 56
.v_w8_loop:
vinserti32x4 m0, [srcq+strideq*1], 3
lea srcq, [srcq+strideq*2]
movu xm5, [srcq+strideq*0]
mova m4, m10
vpdpwssd m4, m12, m1 ; a0 b0
mova m1, m2
vshufi32x4 m0, m5, q1032 ; 6 7 8
vpdpwssd m4, m13, m2 ; a1 b1
mova m2, m3
vpdpwssd m4, m14, m3 ; a2 b2
vpermb m3, m6, m0 ; 67 78
vpdpwssd m4, m15, m3 ; a3 b3
vpermb m4, m11, m4
mova [tmpq], ym4
add tmpq, 32
sub hd, 2
jg .v_w8_loop
RET
.v_w16:
vbroadcasti32x8 m1, [srcq+strideq*1]
vinserti32x8 m0, m1, [srcq+strideq*0], 0
vinserti32x8 m1, [srcq+strideq*2], 1
mova m8, [spel_v_shuf16]
add srcq, r6
movu ym3, [srcq+strideq*0]
vinserti32x8 m3, [srcq+strideq*1], 1
movu ym5, [srcq+strideq*2]
add srcq, r6
vinserti32x8 m5, [srcq+strideq*0], 1
mova m11, [prep_endA]
vpermb m0, m8, m0 ; 01
vpermb m1, m8, m1 ; 12
vpermb m3, m8, m3 ; 34
vpermb m5, m8, m5 ; 56
vpshrdd m2, m1, m3, 16 ; 23
vpshrdd m4, m3, m5, 16 ; 45
.v_w16_loop:
mova m6, m10
mova m7, m10
vpdpwssd m6, m12, m0 ; a0
mova m0, m2
vpdpwssd m7, m12, m1 ; b0
mova m1, m3
vpdpwssd m6, m13, m2 ; a1
mova m2, m4
vpdpwssd m7, m13, m3 ; b1
mova m3, m5
vpdpwssd m6, m14, m4 ; a2
mova m4, m5
vpdpwssd m7, m14, m5 ; b2
movu ym5, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
vinserti32x8 m5, [srcq+strideq*0], 1
vpermb m5, m8, m5 ; 78
vpshrdd m4, m5, 16 ; 67
vpdpwssd m6, m15, m4 ; a3
vpdpwssd m7, m15, m5 ; b3
vpermt2b m6, m11, m7
mova [tmpq], m6
add tmpq, 64
sub hd, 2
jg .v_w16_loop
RET
.v_w32:
.v_w64:
.v_w128:
%if WIN64
PUSH r8
movaps [rsp+stack_offset+8], xmm6
%endif
lea r5, [hq+wq*8-256]
mov r7, srcq
mov r8, tmpq
.v_w32_loop0:
movu m16, [srcq+strideq*0]
movu m17, [srcq+strideq*1]
movu m18, [srcq+strideq*2]
add srcq, r6
movu m19, [srcq+strideq*0]
movu m20, [srcq+strideq*1]
movu m21, [srcq+strideq*2]
add srcq, r6
movu m22, [srcq+strideq*0]
mova m11, [prep_endC]
punpcklwd m0, m16, m17 ; 01l
punpckhwd m16, m17 ; 01h
punpcklwd m1, m17, m18 ; 12l
punpckhwd m17, m18 ; 12h
punpcklwd m2, m18, m19 ; 23l
punpckhwd m18, m19 ; 23h
punpcklwd m3, m19, m20 ; 34l
punpckhwd m19, m20 ; 34h
punpcklwd m4, m20, m21 ; 45l
punpckhwd m20, m21 ; 45h
punpcklwd m5, m21, m22 ; 56l
punpckhwd m21, m22 ; 56h
.v_w32_loop:
mova m6, m10
vpdpwssd m6, m12, m0 ; a0l
mova m8, m10
vpdpwssd m8, m12, m16 ; a0h
mova m7, m10
vpdpwssd m7, m12, m1 ; b0l
mova m9, m10
vpdpwssd m9, m12, m17 ; b0h
mova m0, m2
vpdpwssd m6, m13, m2 ; a1l
mova m16, m18
vpdpwssd m8, m13, m18 ; a1h
mova m1, m3
vpdpwssd m7, m13, m3 ; b1l
mova m17, m19
vpdpwssd m9, m13, m19 ; b1h
mova m2, m4
vpdpwssd m6, m14, m4 ; a2l
mova m18, m20
vpdpwssd m8, m14, m20 ; a2h
mova m3, m5
vpdpwssd m7, m14, m5 ; b2l
mova m19, m21
vpdpwssd m9, m14, m21 ; b2h
movu m21, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
punpcklwd m4, m22, m21 ; 67l
punpckhwd m20, m22, m21 ; 67h
movu m22, [srcq+strideq*0]
vpdpwssd m6, m15, m4 ; a3l
vpdpwssd m8, m15, m20 ; a3h
punpcklwd m5, m21, m22 ; 78l
punpckhwd m21, m22 ; 78h
vpdpwssd m7, m15, m5 ; b3l
vpdpwssd m9, m15, m21 ; b3h
vpermt2b m6, m11, m8
vpermt2b m7, m11, m9
mova [tmpq+wq*0], m6
mova [tmpq+wq*2], m7
lea tmpq, [tmpq+wq*4]
sub hd, 2
jg .v_w32_loop
add r7, 64
add r8, 64
movzx hd, r5b
mov srcq, r7
mov tmpq, r8
sub r5d, 1<<8
jg .v_w32_loop0
%if WIN64
movaps xmm6, [rsp+stack_offset+8]
POP r8
%endif
vzeroupper
RET
.hv:
cmp wd, 4
jg .hv_w8
movzx mxd, mxb
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
mov r5d, r7m
pmovsxbw xmm1, [base+subpel_filters+myq*8]
lea r6, [strideq*3]
sub srcq, 2
shr r5d, 11
sub srcq, r6
psllw xmm0, [base+prep_hv_shift+r5*8]
psllw xmm1, 2
vpbroadcastd m10, [prep_8tap_rnd]
vpbroadcastd ym11, [pd_128]
mova xm21, [prep_endA]
mova [tmpq+ 0], xmm0
mova [tmpq+16], xmm1
vpbroadcastd m8, [tmpq+ 4]
vpbroadcastd m9, [tmpq+ 8]
vpbroadcastd ym12, xmm1
vpbroadcastd ym13, [tmpq+20]
vpbroadcastd ym14, [tmpq+24]
vpbroadcastd ym15, [tmpq+28]
movu xm4, [srcq+strideq*0]
vinserti32x4 ym4, [srcq+strideq*1], 1
vinserti32x4 m4, [srcq+strideq*2], 2
add srcq, r6
vinserti32x4 m4, [srcq+strideq*0], 3 ; 0 1 2 3
movu xm0, [srcq+strideq*1]
vinserti32x4 ym0, [srcq+strideq*2], 1
add srcq, r6
vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6
vbroadcasti32x4 m19, [spel_h_shufA]
vbroadcasti32x4 m20, [spel_h_shufB]
mova ym6, [spel_shuf4a]
mova ym7, [spel_shuf4b]
mova m2, m10
mova m3, m10
pshufb m1, m4, m19
vpdpwssd m2, m8, m1
pshufb m1, m0, m19
vpdpwssd m3, m8, m1
pshufb m4, m20
vpdpwssd m2, m9, m4
pshufb m0, m20
vpdpwssd m3, m9, m0
vpermb m1, m6, m2 ; 01 12
vshufi32x4 m2, m3, q1032
vpermb m3, m6, m3 ; 45 56
vpermb m2, m6, m2 ; 23 34
.hv_w4_loop:
movu xm18, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
vinserti128 ym18, [srcq+strideq*0], 1
mova ym16, ym11
mova ym4, ym10
pshufb ym17, ym18, ym19
vpdpwssd ym16, ym12, ym1 ; a0 b0
vpdpwssd ym4, ym8, ym17
pshufb ym18, ym20
mova ym1, ym2
vpdpwssd ym16, ym13, ym2 ; a1 b1
vpdpwssd ym4, ym9, ym18 ; 7 8
mova ym2, ym3
vpdpwssd ym16, ym14, ym3 ; a2 b2
vpermt2b ym3, ym7, ym4 ; 67 78
vpdpwssd ym16, ym15, ym3 ; a3 b3
vpermb ym16, ym21, ym16
mova [tmpq], xm16
add tmpq, 16
sub hd, 2
jg .hv_w4_loop
vzeroupper
RET
.hv_w8:
shr mxd, 16
pmovsxbw xmm0, [base+subpel_filters+mxq*8]
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
mov r5d, r7m
pmovsxbw xmm1, [base+subpel_filters+myq*8]
lea r6, [strideq*3]
sub srcq, 6
shr r5d, 11
sub srcq, r6
vpbroadcastd m10, [prep_8tap_rnd]
vpbroadcastd m11, [pd_128]
psllw xmm0, [base+prep_hv_shift+r5*8]
psllw xmm1, 2
mova [tmpq+ 0], xmm0
mova [tmpq+16], xmm1
vpbroadcastd m12, xmm0
vpbroadcastd m13, [tmpq+ 4]
vpbroadcastd m14, [tmpq+ 8]
vpbroadcastd m15, [tmpq+12]
vpbroadcastd m16, xmm1
vpbroadcastd m17, [tmpq+20]
vpbroadcastd m18, [tmpq+24]
vpbroadcastd m19, [tmpq+28]
cmp wd, 16
je .hv_w16
jg .hv_w32
WIN64_SPILL_XMM 23
mova m5, [spel_h_shufA]
movu ym0, [srcq+strideq*0]
vinserti32x8 m0, [srcq+strideq*1], 1 ; 0 1
movu ym9, [srcq+strideq*2]
add srcq, r6
vinserti32x8 m9, [srcq+strideq*0], 1 ; 2 3
movu ym20, [srcq+strideq*1]
vinserti32x8 m20, [srcq+strideq*2], 1 ; 4 5
add srcq, r6
movu ym21, [srcq+strideq*0] ; 6
movu m6, [spel_h_shufB]
movu m7, [spel_h_shufC]
mova ym22, [prep_endB]
vpermb m8, m5, m0
mova m1, m10
vpdpwssd m1, m12, m8 ; a0 b0
vpermb m8, m5, m9
mova m2, m10
vpdpwssd m2, m12, m8 ; c0 d0
vpermb m8, m5, m20
mova m3, m10
vpdpwssd m3, m12, m8 ; e0 f0
vpermb m8, m5, m21
mova m4, m10
vpdpwssd m4, m12, m8 ; g0
vpermb m8, m6, m0
vpdpwssd m1, m13, m8 ; a1 b1
vpermb m8, m6, m9
vpdpwssd m2, m13, m8 ; c1 d1
vpermb m8, m6, m20
vpdpwssd m3, m13, m8 ; e1 f1
vpermb m8, m6, m21
vpdpwssd m4, m13, m8 ; g1
vpermb m8, m7, m0
vpdpwssd m1, m14, m8 ; a2 b2
vpermb m8, m7, m9
vpdpwssd m2, m14, m8 ; c2 d2
vpermb m8, m7, m20
vpdpwssd m3, m14, m8 ; e2 f2
vpermb m8, m7, m21
vpdpwssd m4, m14, m8 ; g2
mova m8, [spel_h_shufD]
vpermb m0, m8, m0
vpdpwssd m1, m15, m0 ; a3 b3
mova m0, [spel_shuf8a]
vpermb m9, m8, m9
vpdpwssd m2, m15, m9 ; c3 d3
mova m9, [spel_shuf8b]
vpermb m20, m8, m20
vpdpwssd m3, m15, m20 ; e3 f3
vpermb m21, m8, m21
vpdpwssd m4, m15, m21 ; g3
vpermt2b m1, m0, m2 ; 01 12
vpermt2b m2, m0, m3 ; 23 34
vpermt2b m3, m0, m4 ; 45 56
.hv_w8_loop:
movu ym0, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
vinserti32x8 m0, [srcq+strideq*0], 1
mova m4, m10
mova m20, m11
vpermb m21, m5, m0
vpdpwssd m4, m12, m21 ; h0 i0
vpermb m21, m6, m0
vpdpwssd m20, m16, m1 ; A0 B0
vpdpwssd m4, m13, m21 ; h1 i1
vpermb m21, m7, m0
mova m1, m2
vpdpwssd m20, m17, m2 ; A1 B1
vpdpwssd m4, m14, m21 ; h2 i2
vpermb m21, m8, m0
mova m2, m3
vpdpwssd m20, m18, m3 ; A2 B2
vpdpwssd m4, m15, m21 ; h3 i3
vpermt2b m3, m9, m4 ; 67 78
vpdpwssd m20, m19, m3 ; A3 B3
vpermb m20, m22, m20
mova [tmpq], ym20
add tmpq, 32
sub hd, 2
jg .hv_w8_loop
RET
.hv_w16:
WIN64_SPILL_XMM 27
vbroadcasti32x8 m5, [srcq+strideq*0+ 8]
vinserti32x8 m4, m5, [srcq+strideq*0+ 0], 0
vinserti32x8 m5, [srcq+strideq*0+16], 1 ; 0
movu ym6, [srcq+strideq*1+ 0]
movu ym7, [srcq+strideq*1+16]
vinserti32x8 m6, [srcq+strideq*2+ 0], 1
vinserti32x8 m7, [srcq+strideq*2+16], 1 ; 1 2
add srcq, r6
movu ym22, [srcq+strideq*0+ 0]
movu ym23, [srcq+strideq*0+16]
vinserti32x8 m22, [srcq+strideq*1+ 0], 1
vinserti32x8 m23, [srcq+strideq*1+16], 1 ; 3 4
movu ym24, [srcq+strideq*2+ 0]
movu ym25, [srcq+strideq*2+16]
add srcq, r6
vinserti32x8 m24, [srcq+strideq*0+ 0], 1
vinserti32x8 m25, [srcq+strideq*0+16], 1 ; 5 6
vbroadcasti32x4 m20, [spel_h_shufA]
vbroadcasti32x4 m21, [spel_h_shufB]
mova m9, [spel_shuf16]
mova m26, [prep_endB]
pshufb m0, m4, m20
mova m1, m10
vpdpwssd m1, m12, m0 ; a0
pshufb m0, m6, m20
mova m2, m10
vpdpwssd m2, m12, m0 ; b0
pshufb m0, m7, m20
mova m3, m10
vpdpwssd m3, m14, m0 ; c2
pshufb m0, m4, m21
vpdpwssd m1, m13, m0 ; a1
pshufb m0, m6, m21
vpdpwssd m2, m13, m0 ; b1
pshufb m0, m7, m21
vpdpwssd m3, m15, m0 ; c3
pshufb m0, m5, m20
vpdpwssd m1, m14, m0 ; a2
shufpd m6, m7, 0x55
pshufb m7, m6, m20
vpdpwssd m2, m14, m7 ; b2
vpdpwssd m3, m12, m7 ; c0
pshufb m5, m21
vpdpwssd m1, m15, m5 ; a3
pshufb m6, m21
vpdpwssd m2, m15, m6 ; b3
vpdpwssd m3, m13, m6 ; c1
pshufb m0, m22, m20
mova m4, m10
vpdpwssd m4, m12, m0 ; d0
pshufb m0, m23, m20
mova m5, m10
vpdpwssd m5, m14, m0 ; e2
pshufb m0, m24, m20
mova m6, m10
vpdpwssd m6, m12, m0 ; f0
pshufb m0, m25, m20
mova m7, m10
vpdpwssd m7, m14, m0 ; g2
pshufb m0, m22, m21
vpdpwssd m4, m13, m0 ; d1
pshufb m0, m23, m21
vpdpwssd m5, m15, m0 ; e3
pshufb m0, m24, m21
vpdpwssd m6, m13, m0 ; f1
pshufb m0, m25, m21
vpdpwssd m7, m15, m0 ; g3
shufpd m22, m23, 0x55
pshufb m23, m22, m20
vpdpwssd m4, m14, m23 ; d2
vpdpwssd m5, m12, m23 ; e0
shufpd m24, m25, 0x55
pshufb m25, m24, m20
vpdpwssd m6, m14, m25 ; f2
vpdpwssd m7, m12, m25 ; g0
pshufb m22, m21
vpdpwssd m4, m15, m22 ; d3
vpdpwssd m5, m13, m22 ; e1
pshufb m24, m21
vpdpwssd m6, m15, m24 ; f3
vpdpwssd m7, m13, m24 ; g1
pslldq m1, 1
vpermt2b m2, m9, m3 ; 12
vpermt2b m4, m9, m5 ; 34
vpermt2b m6, m9, m7 ; 56
vpshrdd m1, m2, 16 ; 01
vpshrdd m3, m2, m4, 16 ; 23
vpshrdd m5, m4, m6, 16 ; 45
.hv_w16_loop:
movu ym24, [srcq+strideq*1+ 0]
movu ym25, [srcq+strideq*1+16]
lea srcq, [srcq+strideq*2]
vinserti32x8 m24, [srcq+strideq*0+ 0], 1
vinserti32x8 m25, [srcq+strideq*0+16], 1
mova m7, m10
mova m8, m10
pshufb m0, m24, m20
vpdpwssd m7, m12, m0 ; h0
mova m22, m11
pshufb m0, m25, m20
vpdpwssd m8, m14, m0 ; i2
mova m23, m11
vpdpwssd m22, m16, m1 ; A0
mova m1, m3
vpdpwssd m23, m16, m2 ; B0
mova m2, m4
pshufb m0, m24, m21
vpdpwssd m7, m13, m0 ; h1
pshufb m0, m25, m21
vpdpwssd m8, m15, m0 ; i3
vpdpwssd m22, m17, m3 ; A1
mova m3, m5
vpdpwssd m23, m17, m4 ; B1
mova m4, m6
shufpd m24, m25, 0x55
pshufb m25, m24, m20
vpdpwssd m7, m14, m25 ; h2
vpdpwssd m8, m12, m25 ; i0
vpdpwssd m22, m18, m5 ; A2
vpdpwssd m23, m18, m6 ; B2
pshufb m24, m21
vpdpwssd m7, m15, m24 ; h3
vpdpwssd m8, m13, m24 ; i1
vpermt2b m7, m9, m8 ; 78
vpshrdd m5, m6, m7, 16 ; 67
vpdpwssd m22, m19, m5 ; A3
vpdpwssd m23, m19, m7 ; B3
mova m6, m7
vpermt2b m22, m26, m23
mova [tmpq], m22
add tmpq, 64
sub hd, 2
jg .hv_w16_loop
RET
.hv_w32:
%if WIN64
PUSH r8
%assign regs_used regs_used + 1
WIN64_SPILL_XMM 32
%endif
vbroadcasti32x4 m20, [spel_h_shufA]
vbroadcasti32x4 m21, [spel_h_shufB]
mova m22, [spel_shuf32]
lea r5d, [hq+wq*8-256]
mov r7, srcq
mov r8, tmpq
.hv_w32_loop0:
movu m6, [srcq+strideq*0+ 0]
movu m7, [srcq+strideq*0+ 8]
movu m8, [srcq+strideq*0+16]
mova m0, m10
mova m23, m10
pshufb m9, m6, m20
vpdpwssd m0, m12, m9 ; a0l
pshufb m9, m7, m20
vpdpwssd m23, m12, m9 ; a0h
vpdpwssd m0, m14, m9 ; a2l
pshufb m7, m21
vpdpwssd m23, m13, m7 ; a1h
vpdpwssd m0, m15, m7 ; a3l
pshufb m7, m8, m20
vpdpwssd m23, m14, m7 ; a2h
pshufb m6, m21
vpdpwssd m0, m13, m6 ; a1l
pshufb m8, m21
vpdpwssd m23, m15, m8 ; a3h
PUT_8TAP_HV_W32 1, 24, strideq, 1, 2 ; 12
PUT_8TAP_HV_W32 3, 26, strideq, 0, 1 ; 34
PUT_8TAP_HV_W32 5, 28, strideq, 2, 0 ; 56
vpshrdd m2, m1, m3, 16 ; 23l
vpshrdd m25, m24, m26, 16 ; 23h
vpshrdd m4, m3, m5, 16 ; 45l
vpshrdd m27, m26, m28, 16 ; 45h
.hv_w32_loop:
movu m7, [srcq+strideq*1+ 0]
movu m9, [srcq+strideq*2+ 0]
movu m6, [srcq+strideq*1+ 8]
movu m8, [srcq+strideq*2+ 8]
mova m29, m10
mova m31, m10
pshufb m30, m7, m20
vpdpwssd m29, m12, m30 ; h0l
pshufb m30, m9, m20
vpdpwssd m31, m12, m30 ; i0l
pshufb m7, m21
vpdpwssd m29, m13, m7 ; h1l
pshufb m9, m21
vpdpwssd m31, m13, m9 ; i1l
pshufb m7, m6, m20
vpdpwssd m29, m14, m7 ; h2l
pshufb m9, m8, m20
vpdpwssd m31, m14, m9 ; i2l
pshufb m6, m21
vpdpwssd m29, m15, m6 ; h3l
pshufb m8, m21
vpdpwssd m31, m15, m8 ; i3l
mova m30, m10
vpdpwssd m30, m12, m7 ; h0h
movu m7, [srcq+strideq*1+16]
lea srcq, [srcq+strideq*2]
vpermt2b m29, m22, m31 ; 78l
mova m31, m10
vpdpwssd m31, m12, m9 ; i0h
movu m9, [srcq+strideq*0+16]
vpdpwssd m30, m13, m6 ; h1h
pshufb m6, m7, m20
vpdpwssd m31, m13, m8 ; i1h
pshufb m8, m9, m20
vpdpwssd m30, m14, m6 ; h2h
mova m6, m11
vpdpwssd m6, m16, m0 ; A0l
pshufb m7, m21
vpdpwssd m31, m14, m8 ; i2h
mova m8, m11
vpdpwssd m8, m16, m23 ; A0h
pshufb m9, m21
vpdpwssd m30, m15, m7 ; h3h
mova m7, m11
vpdpwssd m7, m16, m1 ; B0l
vpdpwssd m31, m15, m9 ; i3h
mova m9, m11
vpdpwssd m9, m16, m24 ; B0h
mova m0, m2
vpdpwssd m6, m17, m2 ; A1l
mova m23, m25
vpdpwssd m8, m17, m25 ; A1h
mova m1, m3
vpdpwssd m7, m17, m3 ; B1l
mova m24, m26
vpdpwssd m9, m17, m26 ; B1h
vpermt2b m30, m22, m31 ; 78h
mova m31, [prep_endC]
vpdpwssd m6, m18, m4 ; A2l
mova m2, m4
vpdpwssd m8, m18, m27 ; A2h
mova m25, m27
vpdpwssd m7, m18, m5 ; B2l
mova m3, m5
vpdpwssd m9, m18, m28 ; B2h
mova m26, m28
vpshrdd m4, m5, m29, 16 ; 67l
vpdpwssd m6, m19, m4 ; A3l
vpshrdd m27, m28, m30, 16 ; 67h
vpdpwssd m8, m19, m27 ; A3h
mova m5, m29
vpdpwssd m7, m19, m29 ; B3l
mova m28, m30
vpdpwssd m9, m19, m30 ; B3h
vpermt2b m6, m31, m8
vpermt2b m7, m31, m9
mova [tmpq+wq*0], m6
mova [tmpq+wq*2], m7
lea tmpq, [tmpq+wq*4]
sub hd, 2
jg .hv_w32_loop
add r7, 64
add r8, 64
movzx hd, r5b
mov srcq, r7
mov tmpq, r8
sub r5d, 1<<8
jg .hv_w32_loop0
RET
%if WIN64
DECLARE_REG_TMP 5
%else
DECLARE_REG_TMP 7
%endif
cglobal warp_affine_8x8t_16bpc, 4, 7, 22, tmp, ts
%define base r6-pd_0to7
mov t0d, r7m
lea r6, [pd_0to7]
shr t0d, 11
vpbroadcastd m8, [base+warp_8x8t_rnd_v]
vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4]
call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main
psrad m14, m16, 15
call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2
psrad m16, 15
packssdw m14, m16
call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2
psrad m15, m16, 15
call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2
add tsq, tsq
psrad m16, 15
packssdw m15, m16
jmp mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).end
cglobal warp_affine_8x8_16bpc, 4, 7, 22, dst, ds, src, ss, abcd
mov t0d, r7m ; pixel_max
lea r6, [pd_0to7]
shr t0d, 11
vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4]
vpbroadcastd m8, [base+warp_8x8_rnd_v+t0*4]
call .main
psrad m14, m16, 13
call .main2
psrad m16, 13
packusdw m14, m16
call .main2
psrad m15, m16, 13
call .main2
vpbroadcastd m0, [base+bidir_shift+t0*4]
vpsrlvw m14, m0
psrad m16, 13
packusdw m15, m16
vpsrlvw m15, m0
.end:
mova m0, [base+warp8x8_end]
vpermb m16, m0, m14
lea r2, [dsq*3]
mova [dstq+dsq*0], xm16
vextracti128 [dstq+dsq*1], ym16, 1
vextracti32x4 [dstq+dsq*2], m16, 2
vextracti32x4 [dstq+r2 ], m16, 3
vpermb m16, m0, m15
lea dstq, [dstq+dsq*4]
mova [dstq+dsq*0], xm16
vextracti128 [dstq+dsq*1], ym16, 1
vextracti32x4 [dstq+dsq*2], m16, 2
vextracti32x4 [dstq+r2 ], m16, 3
RET
.main:
vpbroadcastd ym3, [base+pd_512]
%if WIN64
mov abcdq, r5mp
vpaddd ym18, ym3, r6m {1to8} ; mx
%else
add r5d, 512
vpbroadcastd ym18, r5d
%endif
vpaddd ym20, ym3, r7m {1to8} ; my
mova ym16, [base+pd_0to7]
vpbroadcastd ym19, [abcdq+4*0] ; alpha
vpbroadcastd ym21, [abcdq+4*1] ; gamma
lea r4, [ssq*3+6]
vpdpwssd ym18, ym19, ym16 ; tmx
vpdpwssd ym20, ym21, ym16 ; tmy
sub srcq, r4
mova m10, [base+warp8x8_permA]
lea r4, [mc_warp_filter+64*8]
vbroadcasti32x4 m12, [base+warp8x8_permC]
kxnorb k1, k1, k1
vbroadcasti32x4 m13, [base+warp8x8_permD]
movu ym5, [srcq+0]
vinserti32x8 m5, [srcq+8], 1
psrad ym17, ym18, 10
mova m11, [base+warp8x8_permB]
kmovb k2, k1
vpgatherdq m3{k1}, [r4+ym17*8] ; filter_x0
psrad ym19, 16 ; beta
psrad ym21, 16 ; delta
paddd ym18, ym19
vpermb m4, m10, m5
vpbroadcastq m9, [base+warp_shift_h+t0*8]
pshufd m3, m3, q3120
paddd m7, m1, m1
pshufb m2, m3, m12
vpdpwssd m1, m4, m2
vpermb m5, m11, m5
vshufi32x4 m4, m5, q1021
pshufb m3, m13
vpdpwssd m1, m4, m3
call .h
psllq m2, m1, 32
paddd m1, m2
vpmultishiftqb m1, m9, m1
vpshrdq m1, m0, 48 ; 01 12
call .h
vpshrdq m2, m1, m0, 48 ; 23 34
call .h
vpshrdq m3, m2, m0, 48 ; 45 56
.main2:
call .h
psrad ym6, ym20, 10
kmovb k1, k2
paddd ym17, ym20, ym21 ; my += delta
vpgatherdq m20{k2}, [r4+ym6*8] ; filter_y0
psrad ym16, ym17, 10
kmovb k2, k1
vpgatherdq m6{k1}, [r4+ym16*8] ; filter_y1
shufps m5, m20, m6, q2020
mova m16, m8
pshufb m4, m5, m12
vpdpwssd m16, m1, m4 ; a0 b0
pshufb m5, m13
mova m1, m2
vpdpwssd m16, m2, m5 ; a1 b1
shufps m6, m20, m6, q3131
paddd ym20, ym17, ym21
pshufb m4, m6, m12
mova m2, m3
vpdpwssd m16, m3, m4 ; a2 b2
vpshrdq m3, m0, 48 ; 67 78
pshufb m6, m13
vpdpwssd m16, m3, m6 ; a3 b3
ret
ALIGN function_align
.h:
movu ym16, [srcq+ssq*1]
psrad ym6, ym18, 10
lea srcq, [srcq+ssq*2]
vinserti32x8 m5, m16, [srcq+ssq*0], 1
kmovb k1, k2
paddd ym17, ym18, ym19 ; mx += beta
vpgatherdq m18{k2}, [r4+ym6*8] ; filter_x1
psrad ym16, ym17, 10
kmovb k2, k1
vpgatherdq m6{k1}, [r4+ym16*8] ; filter_x2
vpermb m4, m10, m5
shufps m16, m18, m6, q2020
shufps m6, m18, m6, q3131
mova m0, m7
pshufb m18, m16, m12
vpdpwssd m0, m4, m18 ; a0 b0
vpermb m5, m11, m5
pshufb m18, m6, m13
vpdpwssd m0, m5, m18 ; a3 b3
paddd ym18, ym17, ym19
vshufi32x4 m17, m4, m5, q1021
pshufb m16, m13
vpdpwssd m0, m17, m16 ; a1 b1
vshufi32x4 m4, m5, q2132
pshufb m6, m12
vpdpwssd m0, m4, m6 ; a2 b2
vpmultishiftqb m0, m9, m0 ; a a b b
ret
%macro BIDIR_FN 0
call .main
lea stride3q, [strideq*3]
jmp wq
.w4:
movq [dstq ], xm0
movhps [dstq+strideq*1], xm0
vextracti32x4 xm2, ym0, 1
movq [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm2
cmp hd, 8
jl .w4_end
vextracti32x4 xm2, m0, 2
lea dstq, [dstq+strideq*4]
movq [dstq ], xm2
movhps [dstq+strideq*1], xm2
vextracti32x4 xm0, m0, 3
movq [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm0
je .w4_end
lea dstq, [dstq+strideq*4]
movq [dstq ], xm1
movhps [dstq+strideq*1], xm1
vextracti32x4 xm0, ym1, 1
movq [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm0
vextracti32x4 xm0, m1, 2
lea dstq, [dstq+strideq*4]
movq [dstq ], xm0
movhps [dstq+strideq*1], xm0
vextracti32x4 xm1, m1, 3
movq [dstq+strideq*2], xm1
movhps [dstq+stride3q ], xm1
.w4_end:
RET
.w8_loop:
call .main
lea dstq, [dstq+strideq*4]
.w8:
mova [dstq+strideq*0], xm0
vextracti32x4 [dstq+strideq*1], ym0, 1
vextracti32x4 [dstq+strideq*2], m0, 2
vextracti32x4 [dstq+stride3q ], m0, 3
sub hd, 8
jl .w8_end
lea dstq, [dstq+strideq*4]
mova [dstq+strideq*0], xm1
vextracti32x4 [dstq+strideq*1], ym1, 1
vextracti32x4 [dstq+strideq*2], m1, 2
vextracti32x4 [dstq+stride3q ], m1, 3
jg .w8_loop
.w8_end:
RET
.w16_loop:
call .main
lea dstq, [dstq+strideq*4]
.w16:
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
mova [dstq+strideq*2], ym1
vextracti32x8 [dstq+stride3q ], m1, 1
sub hd, 4
jg .w16_loop
RET
.w32_loop:
call .main
lea dstq, [dstq+strideq*2]
.w32:
mova [dstq+strideq*0], m0
mova [dstq+strideq*1], m1
sub hd, 2
jg .w32_loop
RET
.w64_loop:
call .main
add dstq, strideq
.w64:
mova [dstq+64*0], m0
mova [dstq+64*1], m1
dec hd
jg .w64_loop
RET
.w128_loop:
call .main
add dstq, strideq
.w128:
mova [dstq+64*0], m0
mova [dstq+64*1], m1
call .main
mova [dstq+64*2], m0
mova [dstq+64*3], m1
dec hd
jg .w128_loop
RET
%endmacro
%if WIN64
DECLARE_REG_TMP 5
%else
DECLARE_REG_TMP 7
%endif
cglobal avg_16bpc, 4, 7, 4, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-avg_avx512icl_table
lea r6, [avg_avx512icl_table]
tzcnt wd, wm
mov t0d, r6m ; pixel_max
movsxd wq, [r6+wq*4]
shr t0d, 11
vpbroadcastd m2, [base+avg_round+t0*4]
vpbroadcastd m3, [base+avg_shift+t0*4]
movifnidn hd, hm
add wq, r6
BIDIR_FN
ALIGN function_align
.main:
mova m0, [tmp1q+64*0]
paddsw m0, [tmp2q+64*0]
mova m1, [tmp1q+64*1]
paddsw m1, [tmp2q+64*1]
add tmp1q, 64*2
add tmp2q, 64*2
pmaxsw m0, m2
pmaxsw m1, m2
psubsw m0, m2
psubsw m1, m2
vpsrlvw m0, m3
vpsrlvw m1, m3
ret
cglobal w_avg_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-w_avg_avx512icl_table
lea r6, [w_avg_avx512icl_table]
tzcnt wd, wm
mov t0d, r7m ; pixel_max
shr t0d, 11
movsxd wq, [r6+wq*4]
vpbroadcastd m5, [base+w_avg_round+t0*4]
vpbroadcastd m7, [base+bidir_shift+t0*4]
add wq, r6
mov r6d, r6m ; weight
lea t0d, [r6-16]
shl r6d, 16
sub r6d, t0d ; 16-weight, weight
movifnidn hd, hm
vpbroadcastd m6, r6d
BIDIR_FN
ALIGN function_align
.main:
mova m3, [tmp1q+64*0]
mova m1, [tmp2q+64*0]
mova m0, [tmp1q+64*1]
mova m4, [tmp2q+64*1]
add tmp1q, 64*2
add tmp2q, 64*2
punpcklwd m2, m1, m3
punpckhwd m1, m3
punpcklwd m3, m4, m0
punpckhwd m4, m0
mova m0, m5
vpdpwssd m0, m6, m2
mova m2, m5
vpdpwssd m2, m6, m1
mova m1, m5
vpdpwssd m1, m6, m3
mova m3, m5
vpdpwssd m3, m6, m4
REPX {psrad x, 2}, m0, m2, m1, m3
packusdw m0, m2
packusdw m1, m3
vpsrlvw m0, m7
vpsrlvw m1, m7
ret
cglobal mask_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-mask_avx512icl_table
lea r7, [mask_avx512icl_table]
tzcnt wd, wm
mov r6d, r7m ; pixel_max
movifnidn hd, hm
shr r6d, 11
movsxd wq, [r7+wq*4]
vpbroadcastd m8, [base+pw_64]
vpbroadcastd m9, [base+mask_round+r6*4]
vpbroadcastd m10, [base+bidir_shift+r6*4]
mov maskq, maskmp
add wq, r7
BIDIR_FN
ALIGN function_align
.main:
pmovzxbw m1, [maskq+32*0]
mova m4, [tmp1q+64*0]
mova m2, [tmp2q+64*0]
pmovzxbw m6, [maskq+32*1]
mova m5, [tmp1q+64*1]
mova m3, [tmp2q+64*1]
add maskq, 32*2
add tmp1q, 64*2
add tmp2q, 64*2
punpcklwd m7, m4, m2
punpckhwd m4, m2
psubw m0, m8, m1
punpcklwd m2, m1, m0 ; m, 64-m
punpckhwd m1, m0
mova m0, m9
vpdpwssd m0, m7, m2
mova m2, m9
vpdpwssd m2, m4, m1 ; tmp1 * m + tmp2 * (64-m)
punpcklwd m7, m5, m3
punpckhwd m5, m3
psubw m1, m8, m6
punpcklwd m3, m6, m1
punpckhwd m6, m1
mova m1, m9
vpdpwssd m1, m7, m3
mova m3, m9
vpdpwssd m3, m5, m6
REPX {psrad x, 4}, m0, m2, m1, m3
packusdw m0, m2
packusdw m1, m3
vpsrlvw m0, m10
vpsrlvw m1, m10
ret
cglobal w_mask_420_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_420_avx512icl_table
lea r7, [w_mask_420_avx512icl_table]
tzcnt wd, wm
mov r6d, r8m ; pixel_max
movifnidn hd, hm
shr r6d, 11
movsxd wq, [r7+wq*4]
vpbroadcastd m10, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
vpbroadcastd m11, [base+pw_64]
vpbroadcastd m12, [base+mask_round+r6*4]
vpbroadcastd m13, [base+bidir_shift+r6*4]
mov r6d, r7m ; sign
vpbroadcastd m14, [base+w_mask_round+r6*4]
mova ym15, [w_mask_end42x]
mov maskq, maskmp
add wq, r7
call .main
lea stride3q, [strideq*3]
jmp wq
.w4:
mova m4, [w_mask_shuf4]
vpermt2b m2, m4, m3
mova m3, m14
vpdpbusd m3, m2, [pb_64] {1to16}
vpermb m3, m15, m3
movq [dstq+strideq*0], xm0
movhps [dstq+strideq*1], xm0
vextracti32x4 xm2, ym0, 1
movq [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm2
mova [maskq], xm3
cmp hd, 8
jl .w4_end
vextracti32x4 xm2, m0, 2
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm2
movhps [dstq+strideq*1], xm2
vextracti32x4 xm0, m0, 3
movq [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm0
je .w4_end
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm1
movhps [dstq+strideq*1], xm1
vextracti32x4 xm2, ym1, 1
movq [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm2
vextracti32x4 xm2, m1, 2
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm2
movhps [dstq+strideq*1], xm2
vextracti32x4 xm1, m1, 3
movq [dstq+strideq*2], xm1
movhps [dstq+stride3q ], xm1
.w4_end:
RET
.w8:
mova m8, [w_mask_shuf8]
vpbroadcastd m9, [pb_64]
jmp .w8_start
.w8_loop:
call .main
lea dstq, [dstq+strideq*4]
add maskq, 16
.w8_start:
vpermt2b m2, m8, m3
mova m3, m14
vpdpbusd m3, m2, m9
vpermb m3, m15, m3
mova [dstq+strideq*0], xm0
vextracti32x4 [dstq+strideq*1], ym0, 1
vextracti32x4 [dstq+strideq*2], m0, 2
vextracti32x4 [dstq+stride3q ], m0, 3
mova [maskq], xm3
sub hd, 8
jl .w8_end
lea dstq, [dstq+strideq*4]
mova [dstq+strideq*0], xm1
vextracti32x4 [dstq+strideq*1], ym1, 1
vextracti32x4 [dstq+strideq*2], m1, 2
vextracti32x4 [dstq+stride3q ], m1, 3
jg .w8_loop
.w8_end:
RET
.w16:
mova m8, [w_mask_shuf16]
vpbroadcastd m9, [pb_64]
jmp .w16_start
.w16_loop:
call .main
lea dstq, [dstq+strideq*4]
add maskq, 16
.w16_start:
vpermt2b m2, m8, m3
mova m3, m14
vpdpbusd m3, m2, m9
vpermb m3, m15, m3
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
mova [dstq+strideq*2], ym1
vextracti32x8 [dstq+stride3q ], m1, 1
mova [maskq], xm3
sub hd, 4
jg .w16_loop
RET
.w32_loop:
call .main
lea dstq, [dstq+strideq*4]
add maskq, 32
.w32:
paddw m2, m3
mova m8, m14
vpdpwssd m8, m11, m2
mova [dstq+strideq*0], m0
mova [dstq+strideq*1], m1
call .main
paddw m2, m3
mova m3, m14
vpdpwssd m3, m11, m2
vpermt2b m8, m15, m3
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m1
mova [maskq], ym8
sub hd, 4
jg .w32_loop
RET
.w64_loop:
call .main
lea dstq, [dstq+strideq*2]
add maskq, 32
.w64:
mova m8, m2
mova m9, m3
mova [dstq+strideq*0+64*0], m0
mova [dstq+strideq*0+64*1], m1
call .main
paddw m8, m2
paddw m9, m3
mova m2, m14
vpdpwssd m2, m11, m8
mova m3, m14
vpdpwssd m3, m11, m9
vpermt2b m2, m15, m3
mova [dstq+strideq*1+64*0], m0
mova [dstq+strideq*1+64*1], m1
mova [maskq], ym2
sub hd, 2
jg .w64_loop
RET
.w128_loop:
call .main
lea dstq, [dstq+strideq*2]
add maskq, 64
.w128:
mova m16, m2
mova m8, m3
mova [dstq+strideq*0+64*0], m0
mova [dstq+strideq*0+64*1], m1
call .main
mova m17, m2
mova m9, m3
mova [dstq+strideq*0+64*2], m0
mova [dstq+strideq*0+64*3], m1
call .main
paddw m2, m16
paddw m3, m8
mova m16, m14
vpdpwssd m16, m11, m2
mova m8, m14
vpdpwssd m8, m11, m3
mova [dstq+strideq*1+64*0], m0
mova [dstq+strideq*1+64*1], m1
call .main
paddw m2, m17
paddw m3, m9
mova m17, m14
vpdpwssd m17, m11, m2
mova m9, m14
vpdpwssd m9, m11, m3
vpermt2b m16, m15, m8
vpermt2b m17, m15, m9
mova [dstq+strideq*1+64*2], m0
mova [dstq+strideq*1+64*3], m1
mova [maskq+32*0], ym16
mova [maskq+32*1], ym17
sub hd, 2
jg .w128_loop
vzeroupper
RET
ALIGN function_align
.main:
mova m1, [tmp1q+64*0]
mova m3, [tmp2q+64*0]
mova m4, [tmp1q+64*1]
mova m7, [tmp2q+64*1]
add tmp1q, 64*2
add tmp2q, 64*2
psubsw m6, m1, m3
punpcklwd m5, m3, m1
pabsw m6, m6
punpckhwd m3, m1
psubusw m6, m10, m6
psrlw m6, 10 ; 64-m
psubw m2, m11, m6 ; m
punpcklwd m1, m6, m2
punpckhwd m6, m2
mova m0, m12
vpdpwssd m0, m5, m1
mova m1, m12
vpdpwssd m1, m3, m6
psubsw m5, m4, m7
punpcklwd m6, m7, m4
pabsw m5, m5
punpckhwd m7, m4
psubusw m5, m10, m5
psrlw m5, 10
psubw m3, m11, m5
punpcklwd m4, m5, m3
psrad m0, 4
punpckhwd m5, m3
psrad m1, 4
packusdw m0, m1
mova m1, m12
vpdpwssd m1, m6, m4
mova m4, m12
vpdpwssd m4, m7, m5
psrad m1, 4
psrad m4, 4
packusdw m1, m4
vpsrlvw m0, m13
vpsrlvw m1, m13
ret
cglobal w_mask_422_16bpc, 4, 8, 15, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_422_avx512icl_table
lea r7, [w_mask_422_avx512icl_table]
tzcnt wd, wm
mov r6d, r8m ; pixel_max
movifnidn hd, hm
shr r6d, 11
movsxd wq, [r7+wq*4]
vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
vpbroadcastd m9, [base+pw_64]
vpbroadcastd m10, [base+mask_round+r6*4]
vpbroadcastd m11, [base+bidir_shift+r6*4]
mov r6d, r7m ; sign
vpbroadcastd m12, [base+w_mask_round+r6*4]
mova ym13, [w_mask_end42x]
mov maskq, maskmp
add wq, r7
paddw m14, m9, m9 ; pw_128
call .main
lea stride3q, [strideq*3]
jmp wq
.w4:
movq [dstq+strideq*0], xm0
movhps [dstq+strideq*1], xm0
vextracti32x4 xm2, ym0, 1
movq [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm2
cmp hd, 8
jl .w4_end
vextracti32x4 xm2, m0, 2
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm2
movhps [dstq+strideq*1], xm2
vextracti32x4 xm0, m0, 3
movq [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm0
je .w4_end
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm1
movhps [dstq+strideq*1], xm1
vextracti32x4 xm2, ym1, 1
movq [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm2
vextracti32x4 xm2, m1, 2
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm2
movhps [dstq+strideq*1], xm2
vextracti32x4 xm1, m1, 3
movq [dstq+strideq*2], xm1
movhps [dstq+stride3q ], xm1
.w4_end:
RET
.w8_loop:
call .main
lea dstq, [dstq+strideq*4]
.w8:
mova [dstq+strideq*0], xm0
vextracti32x4 [dstq+strideq*1], ym0, 1
vextracti32x4 [dstq+strideq*2], m0, 2
vextracti32x4 [dstq+stride3q ], m0, 3
sub hd, 8
jl .w8_end
lea dstq, [dstq+strideq*4]
mova [dstq+strideq*0], xm1
vextracti32x4 [dstq+strideq*1], ym1, 1
vextracti32x4 [dstq+strideq*2], m1, 2
vextracti32x4 [dstq+stride3q ], m1, 3
jg .w8_loop
.w8_end:
RET
.w16_loop:
call .main
lea dstq, [dstq+strideq*4]
.w16:
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
mova [dstq+strideq*2], ym1
vextracti32x8 [dstq+stride3q ], m1, 1
sub hd, 4
jg .w16_loop
RET
.w32_loop:
call .main
lea dstq, [dstq+strideq*2]
.w32:
mova [dstq+strideq*0], m0
mova [dstq+strideq*1], m1
sub hd, 2
jg .w32_loop
RET
.w64_loop:
call .main
add dstq, strideq
.w64:
mova [dstq+64*0], m0
mova [dstq+64*1], m1
dec hd
jg .w64_loop
RET
.w128_loop:
call .main
add dstq, strideq
.w128:
mova [dstq+64*0], m0
mova [dstq+64*1], m1
call .main
mova [dstq+64*2], m0
mova [dstq+64*3], m1
dec hd
jg .w128_loop
RET
ALIGN function_align
.main:
mova m1, [tmp1q+64*0]
mova m3, [tmp2q+64*0]
mova m4, [tmp1q+64*1]
mova m7, [tmp2q+64*1]
add tmp1q, 64*2
add tmp2q, 64*2
psubsw m6, m1, m3
punpcklwd m5, m3, m1
pabsw m6, m6
punpckhwd m3, m1
psubusw m6, m8, m6
psrlw m6, 10
psubw m2, m9, m6
punpcklwd m1, m6, m2
punpckhwd m6, m2
mova m0, m10
vpdpwssd m0, m5, m1
mova m1, m10
vpdpwssd m1, m3, m6
psubsw m5, m4, m7
punpcklwd m6, m7, m4
pabsw m5, m5
punpckhwd m7, m4
psubusw m5, m8, m5
psrlw m5, 10
psubw m3, m9, m5
punpcklwd m4, m5, m3
psrad m0, 4
punpckhwd m5, m3
psrad m1, 4
packusdw m0, m1
mova m1, m10
vpdpwssd m1, m6, m4
mova m4, m10
vpdpwssd m4, m7, m5
mova m5, m12
vpdpwssd m5, m14, m2
mova m2, m12
vpdpwssd m2, m14, m3
psrad m1, 4
psrad m4, 4
packusdw m1, m4
vpermt2b m5, m13, m2
vpsrlvw m0, m11
vpsrlvw m1, m11
mova [maskq], ym5
add maskq, 32
ret
cglobal w_mask_444_16bpc, 4, 8, 13, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_444_avx512icl_table
lea r7, [w_mask_444_avx512icl_table]
tzcnt wd, wm
mov r6d, r8m ; pixel_max
movifnidn hd, hm
shr r6d, 11
movsxd wq, [r7+wq*4]
vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
vpbroadcastd m9, [base+pw_64]
vpbroadcastd m10, [base+mask_round+r6*4]
mova m11, [w_mask_end444]
vpbroadcastd m12, [base+bidir_shift+r6*4]
mov maskq, maskmp
add wq, r7
call .main
lea stride3q, [strideq*3]
jmp wq
.w4:
movq [dstq+strideq*0], xm0
movhps [dstq+strideq*1], xm0
vextracti32x4 xm2, ym0, 1
movq [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm2
cmp hd, 8
jl .w4_end
vextracti32x4 xm2, m0, 2
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm2
movhps [dstq+strideq*1], xm2
vextracti32x4 xm0, m0, 3
movq [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm0
je .w4_end
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm1
movhps [dstq+strideq*1], xm1
vextracti32x4 xm2, ym1, 1
movq [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm2
vextracti32x4 xm2, m1, 2
lea dstq, [dstq+strideq*4]
movq [dstq+strideq*0], xm2
movhps [dstq+strideq*1], xm2
vextracti32x4 xm1, m1, 3
movq [dstq+strideq*2], xm1
movhps [dstq+stride3q ], xm1
.w4_end:
RET
.w8_loop:
call .main
lea dstq, [dstq+strideq*4]
.w8:
mova [dstq+strideq*0], xm0
vextracti32x4 [dstq+strideq*1], ym0, 1
vextracti32x4 [dstq+strideq*2], m0, 2
vextracti32x4 [dstq+stride3q ], m0, 3
sub hd, 8
jl .w8_end
lea dstq, [dstq+strideq*4]
mova [dstq+strideq*0], xm1
vextracti32x4 [dstq+strideq*1], ym1, 1
vextracti32x4 [dstq+strideq*2], m1, 2
vextracti32x4 [dstq+stride3q ], m1, 3
jg .w8_loop
.w8_end:
RET
.w16_loop:
call .main
lea dstq, [dstq+strideq*4]
.w16:
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
mova [dstq+strideq*2], ym1
vextracti32x8 [dstq+stride3q ], m1, 1
sub hd, 4
jg .w16_loop
RET
.w32_loop:
call .main
lea dstq, [dstq+strideq*2]
.w32:
mova [dstq+strideq*0], m0
mova [dstq+strideq*1], m1
sub hd, 2
jg .w32_loop
RET
.w64_loop:
call .main
add dstq, strideq
.w64:
mova [dstq+64*0], m0
mova [dstq+64*1], m1
dec hd
jg .w64_loop
RET
.w128_loop:
call .main
add dstq, strideq
.w128:
mova [dstq+64*0], m0
mova [dstq+64*1], m1
call .main
mova [dstq+64*2], m0
mova [dstq+64*3], m1
dec hd
jg .w128_loop
RET
ALIGN function_align
.main:
mova m1, [tmp1q+64*0]
mova m3, [tmp2q+64*0]
mova m4, [tmp1q+64*1]
mova m7, [tmp2q+64*1]
add tmp1q, 64*2
add tmp2q, 64*2
psubsw m6, m1, m3
punpcklwd m5, m3, m1
pabsw m6, m6
punpckhwd m3, m1
psubusw m6, m8, m6
psrlw m6, 10
psubw m2, m9, m6
punpcklwd m1, m6, m2
punpckhwd m6, m2
mova m0, m10
vpdpwssd m0, m5, m1
mova m1, m10
vpdpwssd m1, m3, m6
psubsw m5, m4, m7
punpcklwd m6, m7, m4
pabsw m5, m5
punpckhwd m7, m4
psubusw m5, m8, m5
psrlw m5, 10
psubw m3, m9, m5
punpcklwd m4, m5, m3
psrad m0, 4
punpckhwd m5, m3
psrad m1, 4
packusdw m0, m1
mova m1, m10
vpdpwssd m1, m6, m4
mova m4, m10
vpdpwssd m4, m7, m5
vpermt2b m2, m11, m3
psrad m1, 4
psrad m4, 4
packusdw m1, m4
vpsrlvw m0, m12
vpsrlvw m1, m12
mova [maskq], m2
add maskq, 64
ret
cglobal blend_16bpc, 3, 7, 7, dst, ds, tmp, w, h, mask
%define base r6-blend_avx512icl_table
lea r6, [blend_avx512icl_table]
tzcnt wd, wm
movifnidn hd, hm
movsxd wq, [r6+wq*4]
movifnidn maskq, maskmp
vpbroadcastd m6, [base+pw_m512]
add wq, r6
lea r6, [dsq*3]
jmp wq
.w4:
pmovzxbw ym19, [maskq]
movq xm16, [dstq+dsq*0]
movhps xm16, [dstq+dsq*1]
vpbroadcastq ym17, [dstq+dsq*2]
vpbroadcastq ym18, [dstq+r6 ]
pmullw ym19, ym6
vpblendd ym16, ym17, 0x30
vpblendd ym16, ym18, 0xc0
psubw ym17, ym16, [tmpq]
add maskq, 16
add tmpq, 32
pmulhrsw ym17, ym19
paddw ym16, ym17
vextracti128 xm17, ym16, 1
movq [dstq+dsq*0], xm16
movhps [dstq+dsq*1], xm16
movq [dstq+dsq*2], xm17
movhps [dstq+r6 ], xm17
lea dstq, [dstq+dsq*4]
sub hd, 4
jg .w4
vzeroupper
RET
.w8:
pmovzxbw m2, [maskq]
mova xm0, [dstq+dsq*0]
vinserti32x4 ym0, [dstq+dsq*1], 1
vinserti32x4 m0, [dstq+dsq*2], 2
vinserti32x4 m0, [dstq+r6 ], 3
pmullw m2, m6
psubw m1, m0, [tmpq]
add maskq, 32
add tmpq, 64
pmulhrsw m1, m2
paddw m0, m1
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], ym0, 1
vextracti32x4 [dstq+dsq*2], m0, 2
vextracti32x4 [dstq+r6 ], m0, 3
lea dstq, [dstq+dsq*4]
sub hd, 4
jg .w8
RET
.w16:
pmovzxbw m4, [maskq+32*0]
pmovzxbw m5, [maskq+32*1]
mova ym0, [dstq+dsq*0]
vinserti32x8 m0, [dstq+dsq*1], 1
mova ym1, [dstq+dsq*2]
vinserti32x8 m1, [dstq+r6 ], 1
pmullw m4, m6
pmullw m5, m6
psubw m2, m0, [tmpq+64*0]
psubw m3, m1, [tmpq+64*1]
add maskq, 32*2
add tmpq, 64*2
pmulhrsw m2, m4
pmulhrsw m3, m5
paddw m0, m2
paddw m1, m3
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
mova [dstq+dsq*2], ym1
vextracti32x8 [dstq+r6 ], m1, 1
lea dstq, [dstq+dsq*4]
sub hd, 4
jg .w16
RET
.w32:
pmovzxbw m4, [maskq+32*0]
pmovzxbw m5, [maskq+32*1]
mova m0, [dstq+dsq*0]
mova m1, [dstq+dsq*1]
pmullw m4, m6
pmullw m5, m6
psubw m2, m0, [tmpq+ 64*0]
psubw m3, m1, [tmpq+ 64*1]
add maskq, 32*2
add tmpq, 64*2
pmulhrsw m2, m4
pmulhrsw m3, m5
paddw m0, m2
paddw m1, m3
mova [dstq+dsq*0], m0
mova [dstq+dsq*1], m1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w32
RET
cglobal blend_v_16bpc, 3, 6, 5, dst, ds, tmp, w, h
lea r5, [blend_v_avx512icl_table]
tzcnt wd, wm
movifnidn hd, hm
movsxd wq, [r5+wq*4]
add wq, r5
jmp wq
.w2:
vpbroadcastd xmm2, [obmc_masks_avx2+2*2]
.w2_loop:
movd xmm0, [dstq+dsq*0]
pinsrd xmm0, [dstq+dsq*1], 1
movq xmm1, [tmpq]
add tmpq, 4*2
psubw xmm1, xmm0, xmm1
pmulhrsw xmm1, xmm2
paddw xmm0, xmm1
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w2_loop
RET
.w4:
vpbroadcastq xmm2, [obmc_masks_avx2+4*2]
.w4_loop:
movq xmm0, [dstq+dsq*0]
movhps xmm0, [dstq+dsq*1]
psubw xmm1, xmm0, [tmpq]
add tmpq, 8*2
pmulhrsw xmm1, xmm2
paddw xmm0, xmm1
movq [dstq+dsq*0], xmm0
movhps [dstq+dsq*1], xmm0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w4_loop
RET
.w8:
vbroadcasti32x4 ym2, [obmc_masks_avx2+8*2]
.w8_loop:
mova xm0, [dstq+dsq*0]
vinserti32x4 ym0, [dstq+dsq*1], 1
psubw ym1, ym0, [tmpq]
add tmpq, 16*2
pmulhrsw ym1, ym2
paddw ym0, ym1
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], ym0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w8_loop
RET
.w16:
vbroadcasti32x8 m2, [obmc_masks_avx2+16*2]
.w16_loop:
mova ym0, [dstq+dsq*0]
vinserti32x8 m0, [dstq+dsq*1], 1
psubw m1, m0, [tmpq]
add tmpq, 32*2
pmulhrsw m1, m2
paddw m0, m1
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w16_loop
RET
.w32:
mova m4, [obmc_masks_avx2+32*2]
.w32_loop:
mova m0, [dstq+dsq*0]
psubw m2, m0, [tmpq+ 64*0]
mova m1, [dstq+dsq*1]
psubw m3, m1, [tmpq+ 64*1]
add tmpq, 64*2
pmulhrsw m2, m4
pmulhrsw m3, m4
paddw m0, m2
paddw m1, m3
mova [dstq+dsq*0], m0
mova [dstq+dsq*1], m1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w32_loop
RET
cglobal blend_h_16bpc, 3, 7, 9, dst, ds, tmp, w, h, mask
%define base r6-$$
lea r6, [$$]
tzcnt wd, wm
mov hd, hm
movsxd wq, [base+blend_h_avx512icl_table+wq*4]
lea maskq, [base+obmc_masks_avx2+hq*2]
lea hd, [hq*3]
lea wq, [base+blend_h_avx512icl_table+wq]
shr hd, 2 ; h * 3/4
lea maskq, [maskq+hq*2]
neg hq
jmp wq
.w2:
movd xmm0, [dstq+dsq*0]
pinsrd xmm0, [dstq+dsq*1], 1
movd xmm2, [maskq+hq*2]
movq xmm1, [tmpq]
add tmpq, 4*2
punpcklwd xmm2, xmm2
psubw xmm1, xmm0, xmm1
pmulhrsw xmm1, xmm2
paddw xmm0, xmm1
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w2
RET
.w4:
mova xmm3, [blend_shuf]
.w4_loop:
movq xmm0, [dstq+dsq*0]
movhps xmm0, [dstq+dsq*1]
movd xmm2, [maskq+hq*2]
psubw xmm1, xmm0, [tmpq]
add tmpq, 8*2
pshufb xmm2, xmm3
pmulhrsw xmm1, xmm2
paddw xmm0, xmm1
movq [dstq+dsq*0], xmm0
movhps [dstq+dsq*1], xmm0
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w4_loop
RET
.w8:
vbroadcasti32x4 ym3, [blend_shuf]
shufpd ym3, ym3, 0x0c
.w8_loop:
mova xm0, [dstq+dsq*0]
vinserti32x4 ym0, [dstq+dsq*1], 1
vpbroadcastd ym2, [maskq+hq*2]
psubw ym1, ym0, [tmpq]
add tmpq, 16*2
pshufb ym2, ym3
pmulhrsw ym1, ym2
paddw ym0, ym1
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], ym0, 1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w8_loop
RET
.w16:
vbroadcasti32x4 m3, [blend_shuf]
shufpd m3, m3, 0xf0
.w16_loop:
mova ym0, [dstq+dsq*0]
vinserti32x8 m0, [dstq+dsq*1], 1
vpbroadcastd m2, [maskq+hq*2]
psubw m1, m0, [tmpq]
add tmpq, 32*2
pshufb m2, m3
pmulhrsw m1, m2
paddw m0, m1
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w16_loop
RET
.w32:
vpbroadcastw m4, [maskq+hq*2]
vpbroadcastw m5, [maskq+hq*2+2]
mova m0, [dstq+dsq*0]
psubw m2, m0, [tmpq+ 64*0]
mova m1, [dstq+dsq*1]
psubw m3, m1, [tmpq+ 64*1]
add tmpq, 64*2
pmulhrsw m2, m4
pmulhrsw m3, m5
paddw m0, m2
paddw m1, m3
mova [dstq+dsq*0], m0
mova [dstq+dsq*1], m1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w32
RET
.w64:
vpbroadcastw m4, [maskq+hq*2]
mova m0, [dstq+64*0]
psubw m2, m0, [tmpq+64*0]
mova m1, [dstq+64*1]
psubw m3, m1, [tmpq+64*1]
add tmpq, 64*2
pmulhrsw m2, m4
pmulhrsw m3, m4
paddw m0, m2
paddw m1, m3
mova [dstq+64*0], m0
mova [dstq+64*1], m1
add dstq, dsq
inc hq
jl .w64
RET
.w128:
vpbroadcastw m8, [maskq+hq*2]
mova m0, [dstq+64*0]
psubw m4, m0, [tmpq+64*0]
mova m1, [dstq+64*1]
psubw m5, m1, [tmpq+64*1]
mova m2, [dstq+64*2]
psubw m6, m2, [tmpq+64*2]
mova m3, [dstq+64*3]
psubw m7, m3, [tmpq+64*3]
add tmpq, 64*4
REPX {pmulhrsw x, m8}, m4, m5, m6, m7
paddw m0, m4
paddw m1, m5
paddw m2, m6
paddw m3, m7
mova [dstq+64*0], m0
mova [dstq+64*1], m1
mova [dstq+64*2], m2
mova [dstq+64*3], m3
add dstq, dsq
inc hq
jl .w128
RET
cglobal resize_16bpc, 6, 12, 32, dst, dst_stride, src, src_stride, \
dst_w, h, src_w, dx, mx0, pxmax
sub dword mx0m, 4<<14
sub dword src_wm, 8
mov r6, ~0
vpbroadcastd m5, dxm
vpbroadcastd m8, mx0m
vpbroadcastd m6, src_wm
kmovq k6, r6
DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, _, _, pxmax
LEA r7, $$
%define base r7-$$
vpbroadcastd m3, [base+pd_16384]
vpbroadcastd m7, [base+pd_63]
mova m24, [base+resize_permA]
mova m25, [base+resize_permB]
mova m26, [base+resize_permC]
mova m27, [base+resize_permD]
vbroadcasti32x4 m28, [base+resize_shufA]
vbroadcasti32x4 m29, [base+resize_shufB]
mova m30, [base+resize_permE]
vpbroadcastw ym31, pxmaxm
vpdpwssd m8, m5, [base+rescale_mul] ; mx+dx*[0-15]
pslld m5, 4 ; dx*16
pslld m6, 14
pxor m2, m2
.loop_y:
xor xd, xd
mova m4, m8 ; per-line working version of mx
.loop_x:
pmaxsd m0, m4, m2
psrad m9, m4, 8 ; filter offset (unmasked)
pminsd m0, m6 ; iclip(mx, 0, src_w-8)
psubd m1, m4, m0 ; pshufb offset
psrad m0, 14 ; clipped src_x offset
psrad m1, 14 ; pshufb edge_emu offset
vptestmd k5, m1, m1
pand m9, m7 ; filter offset (masked)
ktestw k5, k5
jz .load
vpbroadcastq m14, [base+pd_0_4]
vpermq m10, m0, q1100
vpermq m11, m0, q3322
vpermq m20, m1, q1100
vpermq m21, m1, q3322
punpckldq m10, m10
punpckldq m11, m11
punpckldq m20, m20
punpckldq m21, m21
paddd m10, m14
paddd m11, m14
paddd m20, m14
paddd m21, m14
vextracti32x8 ym12, m10, 1
vextracti32x8 ym13, m11, 1
vextracti32x8 ym22, m20, 1
vextracti32x8 ym23, m21, 1
kmovq k1, k6
kmovq k2, k6
kmovq k3, k6
kmovq k4, k6
vpgatherdq m16{k1}, [srcq+ym10*2] ; 0 1 2 3
vpgatherdq m17{k2}, [srcq+ym11*2] ; 4 5 6 7
vpgatherdq m18{k3}, [srcq+ym12*2] ; 8 9 A B
vpgatherdq m19{k4}, [srcq+ym13*2] ; C D E F
kmovq k1, k6
kmovq k2, k6
kmovq k3, k6
kmovq k4, k6
vpgatherdq m0{k1}, [base+resize_shuf+8+ym20*2]
vpgatherdq m1{k2}, [base+resize_shuf+8+ym21*2]
vpgatherdq m14{k3}, [base+resize_shuf+8+ym22*2]
vpgatherdq m15{k4}, [base+resize_shuf+8+ym23*2]
pshufb m16, m0
pshufb m17, m1
pshufb m18, m14
pshufb m19, m15
mova m20, m24
mova m22, m24
mova m21, m25
mova m23, m25
vpermi2d m20, m16, m17 ; 0-3a 0-3b 4-7a 4-7b
vpermi2d m21, m16, m17 ; 0-3c 0-3d 4-7c 4-7d
vpermi2d m22, m18, m19 ; 8-Ba 8-Bb C-Fa C-Fb
vpermi2d m23, m18, m19 ; 8-Bc 8-Bd C-Fc C-Fd
mova m15, m26
mova m17, m26
mova m16, m27
mova m18, m27
vpermi2q m15, m20, m22 ; 0-3a 4-7a 8-Ba C-Fa
vpermi2q m16, m20, m22 ; 0-3b 4-7b 8-Bb C-Fb
vpermi2q m17, m21, m23 ; 0-3c 4-7c 8-Bc C-Fc
vpermi2q m18, m21, m23 ; 0-3d 4-7d 8-Bd C-Fd
kmovq k1, k6
kmovq k2, k6
vpgatherdd m11{k1}, [base+resize_filter+m9*8+0]
vpgatherdd m13{k2}, [base+resize_filter+m9*8+4]
pshufb m10, m11, m28
pshufb m11, m11, m29
pshufb m12, m13, m28
pshufb m13, m13, m29
jmp .filter
.load:
kmovq k1, k6
kmovq k2, k6
kmovq k3, k6
kmovq k4, k6
vpgatherdd m11{k1}, [base+resize_filter+m9*8+0]
vpgatherdd m13{k2}, [base+resize_filter+m9*8+4]
pshufb m10, m11, m28
pshufb m11, m11, m29
pshufb m12, m13, m28
pshufb m13, m13, m29
vpgatherdd m15{k3}, [srcq+m0*2+ 0]
vpgatherdd m16{k4}, [srcq+m0*2+ 4]
kmovq k1, k6
kmovq k2, k6
vpgatherdd m17{k1}, [srcq+m0*2+ 8]
vpgatherdd m18{k2}, [srcq+m0*2+12]
.filter:
mova m14, m2
vpdpwssd m14, m15, m10
vpdpwssd m14, m16, m11
vpdpwssd m14, m17, m12
vpdpwssd m14, m18, m13
psubd m14, m3, m14
psrad m14, 15
packusdw m14, m14
vpermq m14, m30, m14
pminsw ym14, ym31
mova [dstq+xq*2], ym14
paddd m4, m5
add xd, 16
cmp xd, dst_wd
jl .loop_x
add dstq, dst_strideq
add srcq, src_strideq
dec hd
jg .loop_y
RET
%endif ; ARCH_X86_64