Source code
Revision control
Copy as Markdown
Other Tools
; Copyright © 2022, VideoLAN and dav1d authors
; Copyright © 2022, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
; list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
; this list of conditions and the following disclaimer in the documentation
; and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%include "config.asm"
%include "ext/x86/x86inc.asm"
%include "x86/filmgrain_common.asm"
%if ARCH_X86_64
SECTION_RODATA 16
scale_mask: db -1, -1, 0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1
scale_shift: dw 7, 7, 6, 6, 5, 5, 4, 4
pw_27_17_17_27: dw 108, 68, 68, 108, 27, 17, 17, 27
pw_23_22: dw 92, 88, 0, 128, 23, 22, 0, 32
fg_min: times 2 dw 0
times 2 dw 64
times 2 dw 256
fg_max: times 2 dw 1023
times 2 dw 4095
times 2 dw 960
times 2 dw 3840
times 2 dw 940
times 2 dw 3760
scale_rnd: dd 64
dd 16
uv_offset_mul: dd 256
dd 1024
pb_8_9_0_1: db 8, 9, 0, 1
cextern pb_0to63
SECTION .text
INIT_ZMM avx512icl
cglobal fgy_32x32xn_16bpc, 6, 15, 21, dst, src, stride, fg_data, w, scaling, \
grain_lut, offx, sby, see, offy, src_bak
%define base r11-fg_min
lea r11, [fg_min]
mov r6d, r9m ; bdmax
mov r9d, [fg_dataq+FGData.clip_to_restricted_range]
mov r7d, [fg_dataq+FGData.scaling_shift]
mov sbyd, sbym
vpbroadcastd m6, r9m
shr r6d, 11 ; is_12bpc
vbroadcasti32x4 m7, [base+scale_mask]
shlx r10d, r9d, r6d
vpbroadcastd m10, [base+scale_shift+r7*4-32]
lea r9d, [r6+r9*4]
vpbroadcastd m8, [base+fg_min+r10*4]
kxnorw k1, k1, k1 ; 0xffff
vpbroadcastd m9, [base+fg_max+r9*4]
mov r12, 0xeeeeeeeeeeeeeeee
vpbroadcastd m19, [base+scale_rnd+r6*4]
kshiftrb k2, k1, 4 ; 0xf
vpbroadcastq xm20, [base+pw_27_17_17_27+r6*8]
kmovq k3, r12
vpbroadcastd m11, [base+scale_shift+r6*8+4]
test sbyd, sbyd
setnz r7b
vpbroadcastd m12, [base+pw_27_17_17_27+r6*8+0]
vpbroadcastd m13, [base+pw_27_17_17_27+r6*8+4]
test r7b, [fg_dataq+FGData.overlap_flag]
jnz .v_overlap
imul seed, sbyd, (173 << 24) | 37
add seed, (105 << 24) | 178
rorx seed, seed, 24
movzx seed, seew
xor seed, [fg_dataq+FGData.seed]
lea src_bakq, [srcq+wq*2]
neg wq
sub dstq, srcq
.loop_x:
rorx r6, seeq, 1
or seed, 0xeff4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
rorx offyd, seed, 8
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164
lea offyd, [offyq+offxq*2+747] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
sby, see, offxy, src_bak
mov grain_lutq, grain_lutmp
mov hd, hm
.loop_y:
movu m4, [grain_lutq+offxyq*2+82*0]
movu m5, [grain_lutq+offxyq*2+82*2]
call .add_noise
sub hb, 2
jg .loop_y
add wq, 32
jge .end
lea srcq, [src_bakq+wq*2]
cmp byte [fg_dataq+FGData.overlap_flag], 0
je .loop_x
test sbyd, sbyd
jnz .hv_overlap
; horizontal overlap (without vertical overlap)
.loop_x_h_overlap:
rorx r6, seeq, 1
or seed, 0xeff4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \
sby, see, offy, src_bak, left_offxy
lea left_offxyd, [offyq+73] ; previous column's offy*stride+offx
rorx offyd, seed, 8
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164
lea offyd, [offyq+offxq*2+747] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
sby, see, offxy, src_bak, left_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
.loop_y_h_overlap:
movu m4, [grain_lutq+offxyq*2+82*0]
movu m5, [grain_lutq+offxyq*2+82*2]
movd xm17, [grain_lutq+left_offxyq*2-82*1]
pinsrd xm17, [grain_lutq+left_offxyq*2+82*1], 1
punpckldq xm16, xm4, xm5
punpcklwd xm17, xm16
mova xm16, xm19
vpdpwssd xm16, xm20, xm17
psrad xm16, 1
packssdw xm16, xm16
vpsravw xm16, xm11
vmovdqu8 m4{k2}, m16
vpalignr m5{k2}, m16, m16, 4
call .add_noise
sub hb, 2
jg .loop_y_h_overlap
add wq, 32
jge .end
lea srcq, [src_bakq+wq*2]
test sbyd, sbyd
jnz .hv_overlap
jmp .loop_x_h_overlap
.v_overlap:
movzx sbyd, sbyb
imul seed, [fg_dataq+FGData.seed], 0x00010001
imul r7d, sbyd, 173 * 0x00010001
imul sbyd, 37 * 0x01000100
add r7d, (105 << 16) | 188
add sbyd, (178 << 24) | (141 << 8)
and r7d, 0x00ff00ff
and sbyd, 0xff00ff00
xor seed, r7d
xor seed, sbyd ; (cur_seed << 16) | top_seed
lea src_bakq, [srcq+wq*2]
neg wq
sub dstq, srcq
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \
sby, see, offy, src_bak, _, top_offxy
rorx offyd, seed, 8
rorx offxd, seed, 12
and offyd, 0xf000f
and offxd, 0xf000f
imul offyd, 164
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offyd, [offyq+offxq*2+0x10001*747+32*82]
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
sby, see, offxy, src_bak, _, top_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
movu m16, [grain_lutq+offxyq*2+82*0]
movu m0, [grain_lutq+top_offxyq*2+82*0]
movu m17, [grain_lutq+offxyq*2+82*2]
movu m1, [grain_lutq+top_offxyq*2+82*2]
punpckhwd m4, m0, m16
punpcklwd m0, m16
punpckhwd m5, m1, m17
punpcklwd m1, m17
call .add_noise_v
sub hb, 2
jg .loop_y
add wq, 32
jge .end
lea srcq, [src_bakq+wq*2]
; since fg_dataq.overlap is guaranteed to be set, we never jump back
; to .v_overlap, and instead always fall-through to .hv_overlap
.hv_overlap:
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \
sby, see, offy, src_bak, left_offxy, top_offxy, topleft_offxy
lea topleft_offxyd, [top_offxyq+73]
lea left_offxyd, [offyq+73]
rorx offyd, seed, 8
rorx offxd, seed, 12
and offyd, 0xf000f
and offxd, 0xf000f
imul offyd, 164
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offyd, [offyq+offxq*2+0x10001*747+32*82]
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
sby, see, offxy, src_bak, left_offxy, top_offxy, topleft_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
movu m5, [grain_lutq+offxyq*2+82*0]
movu m0, [grain_lutq+top_offxyq*2+82*0]
movd xm17, [grain_lutq+left_offxyq*2-82*1]
pinsrd xm17, [grain_lutq+topleft_offxyq*2-82*1], 1
movu m2, [grain_lutq+offxyq*2+82*2]
movu m1, [grain_lutq+top_offxyq*2+82*2]
movd xm18, [grain_lutq+left_offxyq*2+82*1]
pinsrd xm18, [grain_lutq+topleft_offxyq*2+82*1], 1
punpckldq xm16, xm5, xm0
punpcklwd xm17, xm16
mova xm16, xm19
vpdpwssd xm16, xm20, xm17
punpckldq xm17, xm2, xm1
punpcklwd xm18, xm17
mova xm17, xm19
vpdpwssd xm17, xm20, xm18
punpckhwd m4, m0, m5
punpcklwd m0, m5
punpckhwd m5, m1, m2
punpcklwd m1, m2
psrad xm16, 1
psrad xm17, 1
packssdw xm16, xm17
vpsravw xm16, xm11
vpshuflw m0{k2}, m16, q1302
punpckhqdq xm16, xm16
vpshuflw m1{k2}, m16, q1302
call .add_noise_v
sub hb, 2
jg .loop_y_h_overlap
add wq, 32
lea srcq, [src_bakq+wq*2]
jl .hv_overlap
.end:
RET
ALIGN function_align
.add_noise_v:
mova m2, m19
vpdpwssd m2, m12, m4
mova m3, m19
vpdpwssd m3, m13, m5
mova m4, m19
vpdpwssd m4, m12, m0
mova m5, m19
vpdpwssd m5, m13, m1
REPX {psrad x, 1}, m2, m3, m4, m5
packssdw m4, m2
packssdw m5, m3
vpsravw m4, m11
vpsravw m5, m11
.add_noise:
mova m0, [srcq+strideq*0]
mova m1, [srcq+strideq*1]
kmovw k4, k1
pand m16, m6, m0
psrld m3, m0, 16
vpgatherdd m2{k4}, [scalingq+m16]
vpcmpud k4, m3, m6, 2 ; px <= bdmax
vpgatherdd m16{k4}, [scalingq+m3]
kmovw k4, k1
pand m17, m6, m1
vpgatherdd m3{k4}, [scalingq+m17]
vpshufb m2{k3}, m16, m7
psrld m16, m1, 16
vpcmpud k4, m16, m6, 2
vpgatherdd m17{k4}, [scalingq+m16]
vpshufb m3{k3}, m17, m7
vpsllvw m2, m10
vpsllvw m3, m10
pmulhrsw m4, m2
pmulhrsw m5, m3
add grain_lutq, 82*4
paddw m0, m4
paddw m1, m5
pmaxsw m0, m8
pmaxsw m1, m8
pminsw m0, m9
pminsw m1, m9
mova [dstq+srcq], m0
add srcq, strideq
mova [dstq+srcq], m1
add srcq, strideq
ret
%macro FGUV_FN 3 ; name, ss_hor, ss_ver
cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 22, dst, src, stride, fg_data, w, scaling, \
grain_lut, h, sby, luma, lstride, uv_pl, is_id
%define base r12-fg_min
lea r12, [fg_min]
mov r9d, r13m ; bdmax
mov r7d, [fg_dataq+FGData.scaling_shift]
mov r6d, [fg_dataq+FGData.clip_to_restricted_range]
mov r11d, is_idm
kxnorw k1, k1, k1 ; 0xffff
vpbroadcastd m5, r13m
mov r13, 0xeeeeeeeeeeeeeeee
vbroadcasti32x4 m6, [base+scale_mask]
shr r9d, 11 ; is_12bpc
vpbroadcastd m7, [base+scale_shift+r7*4-32]
shlx r10d, r6d, r9d
mov sbyd, sbym
shlx r6d, r6d, r11d
vpbroadcastd m8, [base+fg_min+r10*4]
lea r6d, [r9+r6*2]
vpbroadcastd m9, [base+fg_max+r6*4]
kmovq k2, r13
vpbroadcastd m20, [base+scale_rnd+r9*4]
packssdw m4, m5, m5
vpbroadcastd m21, [base+scale_shift+r9*8+4]
%if %2
mova m12, [pb_0to63] ; pw_even
mov r13d, 0x0101
vpbroadcastq m10, [base+pw_23_22+r9*8]
kmovw k3, r13d
%if %3
pshufd m11, m10, q0000
%else
vpbroadcastd ym16, [base+pw_27_17_17_27+r9*8+0]
vpbroadcastd m11, [base+pw_27_17_17_27+r9*8+4]
vmovdqu16 m11{k1}, m16
%endif
psrlw m13, m12, 8 ; pw_odd
%else
vpbroadcastq m10, [base+pw_27_17_17_27+r9*8]
kshiftrb k3, k1, 7 ; 0x01
kshiftrb k4, k1, 4 ; 0x0f
pshufd m11, m10, q0000
%endif
mov lstrideq, r10mp
test sbyd, sbyd
setnz r7b
cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0
jne .csfl
%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
_, sby, see, lstride
%if %1
mov r6d, r11m
vpbroadcastd m0, [base+uv_offset_mul+r9*4]
vpbroadcastd m1, [base+pb_8_9_0_1]
vpbroadcastd m14, [fg_dataq+FGData.uv_offset+r6*4]
vbroadcasti32x4 m15, [fg_dataq+FGData.uv_mult+r6*4]
pmaddwd m14, m0
pshufb m15, m1 ; { uv_luma_mult, uv_mult }
%endif
test r7b, [fg_dataq+FGData.overlap_flag]
jnz %%v_overlap
imul seed, sbyd, (173 << 24) | 37
add seed, (105 << 24) | 178
rorx seed, seed, 24
movzx seed, seew
xor seed, [fg_dataq+FGData.seed]
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
offx, offy, see, lstride, luma
mov lumaq, r9mp
lea r12, [srcq+wq*2]
lea r13, [dstq+wq*2]
lea r14, [lumaq+wq*(2<<%2)]
mov r9mp, r12
mov r10mp, r13
mov r11mp, r14
neg wq
%%loop_x:
rorx r6, seeq, 1
or seed, 0xeff4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
rorx offyd, seed, 8
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164>>%3
lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
h, offxy, see, lstride, luma
mov grain_lutq, grain_lutmp
mov hd, hm
%%loop_y:
%if %2
movu ym18, [grain_lutq+offxyq*2+82*0]
vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
movu ym19, [grain_lutq+offxyq*2+82*4]
vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
%else
movu m18, [grain_lutq+offxyq*2+82*0]
movu m19, [grain_lutq+offxyq*2+82*2]
%endif
call %%add_noise
sub hb, 2<<%2
jg %%loop_y
add wq, 32>>%2
jge .end
mov srcq, r9mp
mov dstq, r10mp
mov lumaq, r11mp
lea srcq, [srcq+wq*2]
lea dstq, [dstq+wq*2]
lea lumaq, [lumaq+wq*(2<<%2)]
cmp byte [fg_dataq+FGData.overlap_flag], 0
je %%loop_x
cmp dword r8m, 0 ; sby
jne %%hv_overlap
; horizontal overlap (without vertical overlap)
%%loop_x_h_overlap:
rorx r6, seeq, 1
or seed, 0xEFF4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
offx, offy, see, lstride, luma, left_offxy
lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx
rorx offyd, seed, 8
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164>>%3
lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
h, offxy, see, lstride, luma, left_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
%%loop_y_h_overlap:
%if %2
movu ym18, [grain_lutq+offxyq*2+82*0]
vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
movu ym19, [grain_lutq+offxyq*2+82*4]
vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
movd xm16, [grain_lutq+left_offxyq*2+82*0]
vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2
movd xm17, [grain_lutq+left_offxyq*2+82*4]
vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2
punpckldq m16, m17
punpckldq m17, m18, m19
punpcklwd m16, m17
mova m17, m20
vpdpwssd m17, m16, m10
psrad m17, 1
packssdw m17, m17
vpsravw m17, m21
%else
movu m18, [grain_lutq+offxyq*2+82*0]
movu m19, [grain_lutq+offxyq*2+82*2]
movd xm16, [grain_lutq+left_offxyq*2+82*0]
pinsrd xm16, [grain_lutq+left_offxyq*2+82*2], 1
punpckldq xm17, xm18, xm19
punpcklwd xm16, xm17
mova xm17, xm20
vpdpwssd xm17, xm16, xm10
psrad xm17, 1
packssdw xm17, xm17
vpsravw xm17, xm21
%endif
vmovdqa32 m18{k3}, m17
vpshufd m19{k3}, m17, q0321
call %%add_noise
sub hb, 2<<%2
jg %%loop_y_h_overlap
add wq, 32>>%2
jge .end
mov srcq, r9mp
mov dstq, r10mp
mov lumaq, r11mp
lea srcq, [srcq+wq*2]
lea dstq, [dstq+wq*2]
lea lumaq, [lumaq+wq*(2<<%2)]
cmp dword r8m, 0 ; sby
jne %%hv_overlap
jmp %%loop_x_h_overlap
%%v_overlap:
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
_, sby, see, lstride
movzx sbyd, sbyb
imul seed, [fg_dataq+FGData.seed], 0x00010001
imul r7d, sbyd, 173 * 0x00010001
imul sbyd, 37 * 0x01000100
add r7d, (105 << 16) | 188
add sbyd, (178 << 24) | (141 << 8)
and r7d, 0x00ff00ff
and sbyd, 0xff00ff00
xor seed, r7d
xor seed, sbyd ; (cur_seed << 16) | top_seed
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
offx, offy, see, lstride, luma, _, top_offxy
mov lumaq, r9mp
lea r12, [srcq+wq*2]
lea r13, [dstq+wq*2]
lea r14, [lumaq+wq*(2<<%2)]
mov r9mp, r12
mov r10mp, r13
mov r11mp, r14
neg wq
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
rorx offyd, seed, 8
rorx offxd, seed, 12
and offyd, 0xf000f
and offxd, 0xf000f
imul offyd, 164>>%3
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
h, offxy, see, lstride, luma, _, top_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
%if %3
movu ym16, [grain_lutq+offxyq*2+82*0]
movu ym1, [grain_lutq+top_offxyq*2+82*0]
vbroadcasti32x8 m18, [grain_lutq+offxyq*2+82*2]
movu ym19, [grain_lutq+offxyq*2+82*4]
vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
punpcklwd ym17, ym1, ym16
punpckhwd ym1, ym16
%elif %2
movu ym18, [grain_lutq+offxyq*2+82*0]
vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
movu ym17, [grain_lutq+top_offxyq*2+82*0]
vinserti32x8 m17, [grain_lutq+top_offxyq*2+82*2], 1
movu ym19, [grain_lutq+offxyq*2+82*4]
vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
punpcklwd m16, m17, m18
punpckhwd m17, m18
%else
movu m18, [grain_lutq+offxyq*2+82*0]
movu m19, [grain_lutq+top_offxyq*2+82*0]
movu m2, [grain_lutq+offxyq*2+82*2]
movu m16, [grain_lutq+top_offxyq*2+82*2]
punpckhwd m1, m19, m18
punpcklwd m19, m18
punpckhwd m18, m2, m16
punpcklwd m2, m16
%endif
call %%add_noise_v
sub hb, 2<<%2
jg %%loop_y
add wq, 32>>%2
jge .end
mov srcq, r9mp
mov dstq, r10mp
mov lumaq, r11mp
lea srcq, [srcq+wq*2]
lea dstq, [dstq+wq*2]
lea lumaq, [lumaq+wq*(2<<%2)]
; since fg_dataq.overlap is guaranteed to be set, we never jump back
; to %%v_overlap, and instead always fall-through to %%hv_overlap
%%hv_overlap:
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
offx, offy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy
lea topleft_offxyq, [top_offxyq+(32>>%2)]
lea left_offxyq, [offyq+(32>>%2)]
rorx offyd, seed, 8
rorx offxd, seed, 12
and offyd, 0xf000f
and offxd, 0xf000f
imul offyd, 164>>%3
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
h, offxy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
; grain = grain_lut[offy+y][offx+x]
%if %2
movd xm16, [grain_lutq+left_offxyq*2+82*0]
vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2
movd xm17, [grain_lutq+left_offxyq*2+82*4]
vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2
movu ym18, [grain_lutq+offxyq*2+82*0]
vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
movu ym19, [grain_lutq+offxyq*2+82*4]
vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
punpckldq m16, m17
punpckldq m17, m18, m19
punpcklwd m16, m17
movu ym1, [grain_lutq+top_offxyq*2+82*0]
movd xm17, [grain_lutq+topleft_offxyq*2+82*0]
mova m0, m20
vpdpwssd m0, m16, m10
%if %3
punpcklwd xm17, xm1
mova xm16, xm20
vpdpwssd xm16, xm17, xm10
psrad xm16, 1
%else
vinserti32x8 m1, [grain_lutq+top_offxyq*2+82*2], 1
vinserti32x4 m17, [grain_lutq+topleft_offxyq*2+82*2], 2
punpcklwd m17, m1
mova m16, m20
vpdpwssd m16, m17, m10
psrad m16, 1
%endif
psrad m0, 1
packssdw m0, m16
vpsravw m0, m21
vmovdqa32 m18{k3}, m0
vpshufd m19{k3}, m0, q0321
%if %3
vpunpckhdq ym1{k3}, ym0, ym0
punpcklwd ym17, ym1, ym18
punpckhwd ym1, ym18
%else
vpunpckhdq m1{k3}, m0, m0
punpcklwd m16, m1, m18
punpckhwd m17, m1, m18
%endif
%else
movu m18, [grain_lutq+offxyq*2+82*0]
movu m19, [grain_lutq+top_offxyq*2+82*0]
movd xm17, [grain_lutq+left_offxyq*2+82*0]
pinsrd xm17, [grain_lutq+topleft_offxyq*2+82*0], 1
punpckldq xm16, xm18, xm19
punpcklwd xm17, xm16
movu m2, [grain_lutq+offxyq*2+82*2]
movu m0, [grain_lutq+top_offxyq*2+82*2]
movd xm16, [grain_lutq+left_offxyq*2+82*2]
pinsrd xm16, [grain_lutq+topleft_offxyq*2+82*2], 1
punpckldq xm1, xm2, xm0
punpcklwd xm1, xm16, xm1
mova xm16, xm20
vpdpwssd xm16, xm17, xm10
mova xm17, xm20
vpdpwssd xm17, xm1, xm10
punpckhwd m1, m19, m18
punpcklwd m19, m18
punpckhwd m18, m2, m0
punpcklwd m2, m0
psrad xm16, 1
psrad xm17, 1
packssdw xm16, xm17
vpsravw xm16, xm21
vpshuflw m19{k4}, m16, q1302
punpckhqdq xm16, xm16
vpshuflw m2{k4}, m16, q3120
%endif
call %%add_noise_v
sub hb, 2<<%2
jg %%loop_y_h_overlap
add wq, 32>>%2
jge .end
mov srcq, r9mp
mov dstq, r10mp
mov lumaq, r11mp
lea srcq, [srcq+wq*2]
lea dstq, [dstq+wq*2]
lea lumaq, [lumaq+wq*(2<<%2)]
jmp %%hv_overlap
ALIGN function_align
%%add_noise_v:
%if %3
mova ym16, ym20
vpdpwssd ym16, ym17, ym11
mova ym17, ym20
vpdpwssd ym17, ym1, ym11
psrad ym16, 1
psrad ym17, 1
packssdw ym16, ym17
vpsravw m18{k1}, m16, m21
%elif %2
mova m18, m20
vpdpwssd m18, m16, m11
mova m16, m20
vpdpwssd m16, m17, m11
psrad m18, 1
psrad m16, 1
packssdw m18, m16
vpsravw m18, m21
%else
mova m16, m20
vpdpwssd m16, m1, m11
mova m17, m20
vpdpwssd m17, m18, m11
mova m18, m20
vpdpwssd m18, m19, m11
mova m19, m20
vpdpwssd m19, m2, m11
REPX {psrad x, 1}, m16, m17, m18, m19
packssdw m18, m16
packssdw m19, m17
vpsravw m18, m21
vpsravw m19, m21
%endif
%%add_noise:
%if %2
mova m2, [lumaq+lstrideq*(0<<%3)]
mova m0, [lumaq+lstrideq*(1<<%3)]
lea lumaq, [lumaq+lstrideq*(2<<%3)]
mova m3, [lumaq+lstrideq*(0<<%3)]
mova m1, [lumaq+lstrideq*(1<<%3)]
mova m16, m12
vpermi2w m16, m2, m0
vpermt2w m2, m13, m0
mova m17, m12
vpermi2w m17, m3, m1
vpermt2w m3, m13, m1
pavgw m2, m16
pavgw m3, m17
%elif %1
mova m2, [lumaq+lstrideq*0]
mova m3, [lumaq+lstrideq*1]
%endif
%if %2
mova ym16, [srcq+strideq*0]
vinserti32x8 m16, [srcq+strideq*1], 1
lea srcq, [srcq+strideq*2]
%else
mova m16, [srcq+strideq*0]
%endif
%if %1
punpckhwd m17, m2, m16
mova m0, m14
vpdpwssd m0, m17, m15
punpcklwd m17, m2, m16
mova m2, m14
vpdpwssd m2, m17, m15
%endif
%if %2
mova ym17, [srcq+strideq*0]
vinserti32x8 m17, [srcq+strideq*1], 1
%else
mova m17, [srcq+strideq*1]
%endif
%if %1
psrad m0, 6
psrad m2, 6
packusdw m2, m0
punpckhwd m0, m3, m17
mova m1, m14
vpdpwssd m1, m15, m0
punpcklwd m0, m3, m17
mova m3, m14
vpdpwssd m3, m15, m0
psrad m1, 6
psrad m3, 6
packusdw m3, m1
pminuw m2, m4
pminuw m3, m4
.add_noise_main:
; scaling[luma_src]
kmovw k5, k1
pand m1, m5, m2
vpgatherdd m0{k5}, [scalingq+m1]
kmovw k5, k1
psrld m2, 16
vpgatherdd m1{k5}, [scalingq+m2]
vpshufb m0{k2}, m1, m6
kmovw k5, k1
psrld m1, m3, 16
vpgatherdd m2{k5}, [scalingq+m1]
kmovw k5, k1
pand m3, m5
vpgatherdd m1{k5}, [scalingq+m3]
vpshufb m1{k2}, m2, m6
; noise = round2(scaling[luma_src] * grain, scaling_shift)
vpsllvw m0, m7
vpsllvw m1, m7
pmulhrsw m18, m0
pmulhrsw m19, m1
add grain_lutq, 82*(4<<%2)
lea lumaq, [lumaq+lstrideq*(2<<%3)]
lea srcq, [srcq+strideq*2]
paddw m16, m18
paddw m17, m19
pmaxsw m16, m8
pmaxsw m17, m8
pminsw m16, m9
pminsw m17, m9
%if %2
mova [dstq+strideq*0], ym16
vextracti32x8 [dstq+strideq*1], m16, 1
lea dstq, [dstq+strideq*2]
mova [dstq+strideq*0], ym17
vextracti32x8 [dstq+strideq*1], m17, 1
%else
mova [dstq+strideq*0], m16
mova [dstq+strideq*1], m17
%endif
lea dstq, [dstq+strideq*2]
ret
%else
%if %2
pand m2, m4
pand m3, m4
%else
pand m2, m4, [lumaq+lstrideq*0]
pand m3, m4, [lumaq+lstrideq*1]
%endif
jmp .add_noise_main
%endif
%endmacro
%%FGUV_32x32xN_LOOP 1, %2, %3
.csfl:
%%FGUV_32x32xN_LOOP 0, %2, %3
.end:
RET
%endmacro
FGUV_FN 420, 1, 1
FGUV_FN 422, 1, 0
FGUV_FN 444, 0, 0
%endif