Source code
Revision control
Copy as Markdown
Other Tools
; Copyright © 2022, VideoLAN and dav1d authors
; Copyright © 2022, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
; list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
; this list of conditions and the following disclaimer in the documentation
; and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%include "config.asm"
%include "ext/x86/x86inc.asm"
%include "x86/filmgrain_common.asm"
%if ARCH_X86_64
SECTION_RODATA 64
pb_even: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94
db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126
pb_odd: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63
db 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95
db 97, 99,101,103,105,107,109,111,113,115,117,119,121,123,125,127
interleave_hl: db 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7
pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32
pb_23_22_0_32: db 23, 22, 0, 32, 0, 32, 0, 32
pb_27_17: times 2 db 27, 17
pb_23_22: times 2 db 23, 22
pw_8: times 2 dw 8
pw_1024: times 2 dw 1024
pb_17_27: times 2 db 17, 27
fg_max: times 4 db 255
times 4 db 240
times 4 db 235
fg_min: times 4 db 0
times 4 db 16
noise_rnd: times 2 dw 128
times 2 dw 64
times 2 dw 32
times 2 dw 16
SECTION .text
INIT_ZMM avx512icl
cglobal fgy_32x32xn_8bpc, 6, 13, 22, dst, src, stride, fg_data, w, scaling, \
grain_lut, h, sby, see, overlap
%define base r11-fg_min
lea r11, [fg_min]
mov r6d, [fg_dataq+FGData.scaling_shift]
mov r7d, [fg_dataq+FGData.clip_to_restricted_range]
mov sbyd, sbym
mov overlapd, [fg_dataq+FGData.overlap_flag]
mov r12, 0x0000000f0000000f ; h_overlap mask
mova m0, [scalingq+64*0]
mova m1, [scalingq+64*1]
mova m2, [scalingq+64*2]
mova m3, [scalingq+64*3]
kmovq k1, r12
vbroadcasti32x4 m4, [base+interleave_hl]
vpbroadcastd ym16, [base+pb_27_17]
vpbroadcastd m12, [base+pb_17_27]
vpbroadcastd m6, [base+noise_rnd+r6*4-32]
test sbyd, sbyd
setnz r6b
vpbroadcastd m7, [base+fg_min+r7*4]
vpbroadcastd m8, [base+fg_max+r7*8]
pxor m5, m5
vpbroadcastd m9, [base+pw_1024]
vpbroadcastq m10, [base+pb_27_17_17_27]
vmovdqa64 m12{k1}, m16
test r6b, overlapb
jnz .v_overlap
imul seed, sbyd, (173 << 24) | 37
add seed, (105 << 24) | 178
rorx seed, seed, 24
movzx seed, seew
xor seed, [fg_dataq+FGData.seed]
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
h, sby, see, overlap
lea src_bakq, [srcq+wq]
neg wq
sub dstq, srcq
.loop_x:
rorx r6, seeq, 1
or seed, 0xeff4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
rorx offyd, seed, 8
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164
lea offxd, [offyq+offxq*2+829] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
h, sby, see, overlap
mov grain_lutq, grain_lutmp
mov hd, hm
.loop_y:
movu ym21, [grain_lutq+offxyq-82]
vinserti32x8 m21, [grain_lutq+offxyq+ 0], 1
call .add_noise
sub hb, 2
jg .loop_y
add wq, 32
jge .end
lea srcq, [src_bakq+wq]
test overlapd, overlapd
jz .loop_x
test sbyd, sbyd
jnz .hv_overlap
.loop_x_h_overlap:
rorx r6, seeq, 1
or seed, 0xeff4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
h, sby, see, left_offxy
rorx offyd, seed, 8
mov left_offxyd, offxd ; previous column's offy*stride
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164
lea offxd, [offyq+offxq*2+829] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
h, sby, see, left_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
.loop_y_h_overlap:
movu ym20, [grain_lutq+offxyq-82]
vinserti32x8 m20, [grain_lutq+offxyq+ 0], 1
movd xm19, [grain_lutq+left_offxyq-50]
vinserti32x4 m19, [grain_lutq+left_offxyq+32], 2
punpcklbw m19, m20
pmaddubsw m19, m10, m19
pmulhrsw m19, m9
punpckhbw m21, m20, m5
packsswb m20{k1}, m19, m19
punpcklbw m20, m5, m20
call .add_noise_h
sub hb, 2
jg .loop_y_h_overlap
add wq, 32
jge .end
lea srcq, [src_bakq+wq]
test sbyd, sbyd
jnz .hv_overlap
jmp .loop_x_h_overlap
.v_overlap:
DEFINE_ARGS dst, src, stride, fg_data, w, offy, offx, \
h, sby, see, overlap
movzx r6d, sbyb
imul seed, [fg_dataq+FGData.seed], 0x00010001
imul r7d, r6d, 173 * 0x00010001
imul r6d, 37 * 0x01000100
add r7d, (105 << 16) | 188
add r6d, (178 << 24) | (141 << 8)
and r7d, 0x00ff00ff
and r6d, 0xff00ff00
xor seed, r7d
xor seed, r6d ; (cur_seed << 16) | top_seed
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
h, sby, see, overlap
lea src_bakq, [srcq+wq]
neg wq
sub dstq, srcq
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
rorx offyd, seed, 8
rorx offxd, seed, 12
and offyd, 0xf000f
and offxd, 0xf000f
imul offyd, 164
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offxd, [offyq+offxq*2+0x10001*829+32*82]
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
h, sby, see, overlap, top_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
movu ym19, [grain_lutq+offxyq-82]
vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1
movu ym21, [grain_lutq+top_offxyq-82]
vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1
punpckhbw m20, m21, m19
punpcklbw m21, m19
call .add_noise_v
sub hb, 2
jg .loop_y
add wq, 32
jge .end
lea srcq, [src_bakq+wq]
; since fg_dataq.overlap is guaranteed to be set, we never jump back
; to .v_overlap, and instead always fall-through to h+v overlap
.hv_overlap:
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
h, sby, see, left_offxy, top_offxy, topleft_offxy
mov topleft_offxyd, top_offxyd
rorx offyd, seed, 8
mov left_offxyd, offxd
rorx offxd, seed, 12
and offyd, 0xf000f
and offxd, 0xf000f
imul offyd, 164
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offxd, [offyq+offxq*2+0x10001*829+32*82]
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
h, sby, see, left_offxy, top_offxy, topleft_offxy
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
movu ym19, [grain_lutq+offxyq-82]
vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1
movd xm16, [grain_lutq+left_offxyq-50]
vinserti32x4 m16, [grain_lutq+left_offxyq+32], 2
movu ym21, [grain_lutq+top_offxyq-82]
vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1
movd xm17, [grain_lutq+topleft_offxyq-50]
vinserti32x4 m17, [grain_lutq+topleft_offxyq+32], 2
; do h interpolation first (so top | top/left -> top, left | cur -> cur)
punpcklbw m16, m19
pmaddubsw m16, m10, m16
punpcklbw m17, m21
pmaddubsw m17, m10, m17
punpckhbw m20, m21, m19
pmulhrsw m16, m9
pmulhrsw m17, m9
packsswb m19{k1}, m16, m16
packsswb m21{k1}, m17, m17
; followed by v interpolation (top | cur -> cur)
punpcklbw m21, m19
call .add_noise_v
sub hb, 2
jg .loop_y_h_overlap
add wq, 32
lea srcq, [src_bakq+wq]
jl .hv_overlap
.end:
RET
ALIGN function_align
.add_noise_v:
pmaddubsw m20, m12, m20
pmaddubsw m21, m12, m21
pmulhrsw m20, m9
pmulhrsw m21, m9
packsswb m21, m20
.add_noise:
punpcklbw m20, m5, m21
punpckhbw m21, m5
.add_noise_h:
mova ym18, [srcq+strideq*0]
vinserti32x8 m18, [srcq+strideq*1], 1
mova m19, m0
punpcklbw m16, m18, m5
vpermt2b m19, m18, m1 ; scaling[ 0..127]
vpmovb2m k2, m18
punpckhbw m17, m18, m5
vpermi2b m18, m2, m3 ; scaling[128..255]
vmovdqu8 m19{k2}, m18 ; scaling[src]
pshufb m19, m4
pmaddubsw m18, m19, m20
pmaddubsw m19, m21
add grain_lutq, 82*2
pmulhrsw m18, m6 ; noise
pmulhrsw m19, m6
paddw m16, m18
paddw m17, m19
packuswb m16, m17
pmaxub m16, m7
pminub m16, m8
mova [dstq+srcq], ym16
add srcq, strideq
vextracti32x8 [dstq+srcq], m16, 1
add srcq, strideq
ret
%macro FGUV_FN 3 ; name, ss_hor, ss_ver
cglobal fguv_32x32xn_i%1_8bpc, 6, 14+%2, 22, dst, src, stride, fg_data, w, \
scaling, grain_lut, h, sby, luma, \
overlap, uv_pl, is_id, _, stride3
lea r11, [fg_min]
mov r6d, [fg_dataq+FGData.scaling_shift]
mov r7d, [fg_dataq+FGData.clip_to_restricted_range]
mov r9d, is_idm
mov sbyd, sbym
mov overlapd, [fg_dataq+FGData.overlap_flag]
%if %2
mov r12, 0x000f000f000f000f ; h_overlap mask
vpbroadcastq m10, [base+pb_23_22_0_32]
lea stride3q, [strideq*3]
%else
mov r12, 0x0000000f0000000f
vpbroadcastq m10, [base+pb_27_17_17_27]
%endif
mova m0, [scalingq+64*0]
mova m1, [scalingq+64*1]
mova m2, [scalingq+64*2]
mova m3, [scalingq+64*3]
kmovq k1, r12
vbroadcasti32x4 m4, [base+interleave_hl]
vpbroadcastd m6, [base+noise_rnd+r6*4-32]
vpbroadcastd m7, [base+fg_min+r7*4]
shlx r7d, r7d, r9d
vpbroadcastd m8, [base+fg_max+r7*4]
test sbyd, sbyd
setnz r7b
vpbroadcastd m9, [base+pw_1024]
mova m11, [base+pb_even]
mova m12, [base+pb_odd]
pxor m5, m5
mov r5, r10mp ; lstride
cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0
jne .csfl
%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver
DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \
h, sby, see, overlap, uv_pl, _, _, stride3
%if %1
mov r6d, uv_plm
vpbroadcastd m16, [base+pw_8]
vbroadcasti32x4 m14, [fg_dataq+FGData.uv_mult+r6*4]
vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4]
pshufb m14, m16 ; uv_luma_mult, uv_mult
%endif
test r7b, overlapb
jnz %%v_overlap
imul seed, sbyd, (173 << 24) | 37
add seed, (105 << 24) | 178
rorx seed, seed, 24
movzx seed, seew
xor seed, [fg_dataq+FGData.seed]
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
offx, offy, see, overlap, _, _, _, stride3
mov lumaq, r9mp
lea r11, [srcq+wq]
lea r12, [dstq+wq]
lea r13, [lumaq+wq*(1+%2)]
mov r11mp, r11
mov r12mp, r12
neg wq
%%loop_x:
rorx r6, seeq, 1
or seed, 0xeff4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
rorx offyd, seed, 8
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164>>%3
lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
h, offxy, see, overlap, _, _, _, stride3
mov grain_lutq, grain_lutmp
mov hd, hm
%%loop_y:
%if %2
movu xm21, [grain_lutq+offxyq+82*0]
vinserti128 ym21, [grain_lutq+offxyq+82*1], 1
vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
%else
movu ym21, [grain_lutq+offxyq+82*0]
vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1
%endif
call %%add_noise
sub hb, 2<<%2
jg %%loop_y
add wq, 32>>%2
jge .end
mov srcq, r11mp
mov dstq, r12mp
lea lumaq, [r13+wq*(1<<%2)]
add srcq, wq
add dstq, wq
test overlapd, overlapd
jz %%loop_x
cmp dword r8m, 0 ; sby
jne %%hv_overlap
; horizontal overlap (without vertical overlap)
%%loop_x_h_overlap:
rorx r6, seeq, 1
or seed, 0xeff4
test seeb, seeh
lea seed, [r6+0x8000]
cmovp seed, r6d ; updated seed
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
offx, offy, see, left_offxy, _, _, _, stride3
lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx
rorx offyd, seed, 8
rorx offxq, seeq, 12
and offyd, 0xf
imul offyd, 164>>%3
lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
h, offxy, see, left_offxy, _, _, _, stride3
mov grain_lutq, grain_lutmp
mov hd, hm
%%loop_y_h_overlap:
%if %2
movu xm20, [grain_lutq+offxyq +82*0]
movd xm19, [grain_lutq+left_offxyq+82*0]
vinserti32x4 ym20, [grain_lutq+offxyq +82*1], 1
vinserti32x4 ym19, [grain_lutq+left_offxyq+82*1], 1
vinserti32x4 m20, [grain_lutq+offxyq +82*2], 2
vinserti32x4 m19, [grain_lutq+left_offxyq+82*2], 2
vinserti32x4 m20, [grain_lutq+offxyq +82*3], 3
vinserti32x4 m19, [grain_lutq+left_offxyq+82*3], 3
%else
movu ym20, [grain_lutq+offxyq + 0]
movd xm19, [grain_lutq+left_offxyq+ 0]
vinserti32x8 m20, [grain_lutq+offxyq +82], 1
vinserti32x4 m19, [grain_lutq+left_offxyq+82], 2
%endif
punpcklbw m19, m20
pmaddubsw m19, m10, m19
punpckhbw m21, m20, m5
pmulhrsw m19, m9
vpacksswb m20{k1}, m19, m19
punpcklbw m20, m5, m20
call %%add_noise_h
sub hb, 2<<%2
jg %%loop_y_h_overlap
add wq, 32>>%2
jge .end
mov srcq, r11mp
mov dstq, r12mp
lea lumaq, [r13+wq*(1<<%2)]
add srcq, wq
add dstq, wq
cmp dword r8m, 0 ; sby
jne %%hv_overlap
jmp %%loop_x_h_overlap
%%v_overlap:
DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \
_, sby, see, overlap, _, _, _, stride3
movzx sbyd, sbyb
imul seed, [fg_dataq+FGData.seed], 0x00010001
imul r7d, sbyd, 173 * 0x00010001
imul sbyd, 37 * 0x01000100
add r7d, (105 << 16) | 188
add sbyd, (178 << 24) | (141 << 8)
and r7d, 0x00ff00ff
and sbyd, 0xff00ff00
xor seed, r7d
xor seed, sbyd ; (cur_seed << 16) | top_seed
%if %3
vpbroadcastd m13, [base+pb_23_22]
kxnorw k3, k3, k3 ; v_overlap mask
%elif %2
vbroadcasti32x8 m13, [base+pb_27_17]
kxnord k3, k3, k3
pshufd m13, m13, q0000 ; 8x27_17, 8x17_27
%else
vpbroadcastd ym16, [base+pb_27_17]
vpbroadcastd m13, [base+pb_17_27]
vmovdqa64 m13{k1}, m16
%endif
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
offx, offy, see, overlap, top_offxy, _, _, stride3
mov lumaq, r9mp
lea r11, [srcq+wq]
lea r12, [dstq+wq]
lea r13, [lumaq+wq*(1<<%2)]
mov r11mp, r11
mov r12mp, r12
neg wq
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
rorx offyd, seed, 8
rorx offxd, seed, 12
and offyd, 0x000f000f
and offxd, 0x000f000f
imul offyd, 164>>%3
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
h, offxy, see, overlap, top_offxy, _, _, stride3
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
%if %3
movu xm18, [grain_lutq+offxyq+82*0]
movu xm20, [grain_lutq+top_offxyq+82*0]
; only interpolate first line, insert remaining line unmodified
vbroadcasti128 ym21, [grain_lutq+offxyq+82*1]
vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
punpcklbw xm19, xm20, xm18
punpckhbw xm20, xm18
%elif %2
movu xm18, [grain_lutq+offxyq+82*0]
vinserti128 ym18, [grain_lutq+offxyq+82*1], 1
movu xm20, [grain_lutq+top_offxyq+82*0]
vinserti32x4 ym20, [grain_lutq+top_offxyq+82*1], 1
vbroadcasti32x4 m21, [grain_lutq+offxyq+82*2]
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
punpcklbw ym19, ym20, ym18
punpckhbw ym20, ym18
%else
movu ym21, [grain_lutq+offxyq+82*0]
vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1
movu ym20, [grain_lutq+top_offxyq+82*0]
vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1
%endif
call %%add_noise_v
sub hb, 2<<%2
jg %%loop_y
add wq, 32>>%2
jge .end
mov srcq, r11mp
mov dstq, r12mp
lea lumaq, [r13+wq*(1<<%2)]
add srcq, wq
add dstq, wq
%%hv_overlap:
; we assume from the block above that bits 8-15 of r7d are zero'ed
mov r6d, seed
or seed, 0xeff4eff4
test seeb, seeh
setp r7b ; parity of top_seed
shr seed, 16
shl r7d, 16
test seeb, seeh
setp r7b ; parity of cur_seed
or r6d, 0x00010001
xor r7d, r6d
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
offx, offy, see, left_offxy, top_offxy, topleft_offxy, _, stride3
lea topleft_offxyd, [top_offxyq+(32>>%2)]
lea left_offxyd, [offyq+(32>>%2)]
rorx offyd, seed, 8
rorx offxd, seed, 12
and offyd, 0x000f000f
and offxd, 0x000f000f
imul offyd, 164>>%3
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
h, offxy, see, left_offxy, top_offxy, topleft_offxy, _, stride3
mov grain_lutq, grain_lutmp
mov hd, hm
movzx top_offxyd, offxyw
shr offxyd, 16
%if %2
movu xm21, [grain_lutq+offxyq+82*0]
movd xm16, [grain_lutq+left_offxyq+82*0]
vinserti128 ym21, [grain_lutq+offxyq+82*1], 1
vinserti128 ym16, [grain_lutq+left_offxyq+82*1], 1
vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2
vinserti32x4 m16, [grain_lutq+left_offxyq+82*2], 2
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
vinserti32x4 m16, [grain_lutq+left_offxyq+82*3], 3
movd xm18, [grain_lutq+topleft_offxyq+82*0]
movu xm20, [grain_lutq+top_offxyq]
; do h interpolation first (so top | top/left -> top, left | cur -> cur)
punpcklbw m16, m21
%if %3
punpcklbw xm18, xm20
%else
vinserti128 ym18, [grain_lutq+topleft_offxyq+82*1], 1
vinserti128 ym20, [grain_lutq+top_offxyq+82*1], 1
punpcklbw ym18, ym20
%endif
punpcklqdq m16, m18
pmaddubsw m16, m10, m16
pmulhrsw m16, m9
packsswb m16, m16
vmovdqu8 m21{k1}, m16
%if %3
vpalignr xm20{k1}, xm16, xm16, 4
punpcklbw xm19, xm20, xm21
punpckhbw xm20, xm21
%else
vpalignr ym20{k1}, ym16, ym16, 4
punpcklbw ym19, ym20, ym21
punpckhbw ym20, ym21
%endif
%else
movu ym21, [grain_lutq+offxyq+82*0]
vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1
movd xm16, [grain_lutq+left_offxyq+82*0]
vinserti32x4 m16, [grain_lutq+left_offxyq+82*1], 2
movu ym20, [grain_lutq+top_offxyq+82*0]
vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1
movd xm18, [grain_lutq+topleft_offxyq+82*0]
vinserti32x4 m18, [grain_lutq+topleft_offxyq+82*1], 2
punpcklbw m16, m21
punpcklbw m18, m20
punpcklqdq m16, m18
pmaddubsw m16, m10, m16
pmulhrsw m16, m9
packsswb m16, m16
vpalignr m20{k1}, m16, m16, 4
vmovdqu8 m21{k1}, m16
%endif
call %%add_noise_v
sub hb, 2<<%2
jg %%loop_y_h_overlap
add wq, 32>>%2
jge .end
mov srcq, r11mp
mov dstq, r12mp
lea lumaq, [r13+wq*(1<<%2)]
add srcq, wq
add dstq, wq
jmp %%hv_overlap
ALIGN function_align
%%add_noise_v:
%if %3
pmaddubsw xm19, xm13, xm19
pmaddubsw xm20, xm13, xm20
pmulhrsw xm19, xm9
pmulhrsw xm20, xm9
vpacksswb m21{k3}, m19, m20
%elif %2
pmaddubsw ym19, ym13, ym19
pmaddubsw ym20, ym13, ym20
pmulhrsw ym19, ym9
pmulhrsw ym20, ym9
vpacksswb m21{k3}, m19, m20
%else
punpcklbw m19, m20, m21
punpckhbw m20, m21
pmaddubsw m19, m13, m19
pmaddubsw m20, m13, m20
pmulhrsw m19, m9
pmulhrsw m20, m9
packsswb m21, m19, m20
%endif
%%add_noise:
punpcklbw m20, m5, m21
punpckhbw m21, m5
%%add_noise_h:
mova ym18, [lumaq+lstrideq*(0<<%3)]
vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1
%if %2
lea lumaq, [lumaq+lstrideq*(2<<%3)]
mova ym16, [lumaq+lstrideq*(0<<%3)]
vinserti32x8 m16, [lumaq+lstrideq*(1<<%3)], 1
mova xm17, [srcq+strideq*0]
mova m19, m11
vpermi2b m19, m18, m16
vinserti128 ym17, [srcq+strideq*1], 1
vpermt2b m18, m12, m16
vinserti32x4 m17, [srcq+strideq*2], 2
pavgb m18, m19
vinserti32x4 m17, [srcq+stride3q ], 3
%else
mova ym17, [srcq+strideq*0]
vinserti32x8 m17, [srcq+strideq*1], 1
%endif
%if %1
punpckhbw m19, m18, m17
punpcklbw m18, m17 ; { luma, chroma }
pmaddubsw m19, m14
pmaddubsw m18, m14
psraw m19, 6
psraw m18, 6
paddw m19, m15
paddw m18, m15
packuswb m18, m19
.add_noise_main:
mova m19, m0
vpermt2b m19, m18, m1 ; scaling[ 0..127]
vpmovb2m k2, m18
vpermi2b m18, m2, m3 ; scaling[128..255]
vmovdqu8 m19{k2}, m18 ; scaling[src]
pshufb m19, m4
pmaddubsw m18, m19, m20
pmaddubsw m19, m21
add grain_lutq, 82*2<<%2
lea lumaq, [lumaq+lstrideq*(2<<%3)]
lea srcq, [srcq+strideq*(2<<%2)]
pmulhrsw m18, m6 ; noise
pmulhrsw m19, m6
punpcklbw m16, m17, m5 ; chroma
punpckhbw m17, m5
paddw m16, m18
paddw m17, m19
packuswb m16, m17
pmaxub m16, m7
pminub m16, m8
%if %2
mova [dstq+strideq*0], xm16
vextracti128 [dstq+strideq*1], ym16, 1
vextracti32x4 [dstq+strideq*2], m16, 2
vextracti32x4 [dstq+stride3q ], m16, 3
%else
mova [dstq+strideq*0], ym16
vextracti32x8 [dstq+strideq*1], m16, 1
%endif
lea dstq, [dstq+strideq*(2<<%2)]
ret
%else
jmp .add_noise_main
%endif
%endmacro
%%FGUV_32x32xN_LOOP 1, %2, %3
.csfl:
%%FGUV_32x32xN_LOOP 0, %2, %3
.end:
RET
%endmacro
FGUV_FN 420, 1, 1
FGUV_FN 422, 1, 0
FGUV_FN 444, 0, 0
%endif ; ARCH_X86_64