// Code generated by command: go run gen.go -avx -out ../accum_vector_avx_amd64.s -pkg xxh3. DO NOT EDIT.

#include "textflag.h"

DATA prime_avx<>+0(SB)/8, $0x000000009e3779b1
DATA prime_avx<>+8(SB)/8, $0x000000009e3779b1
DATA prime_avx<>+16(SB)/8, $0x000000009e3779b1
DATA prime_avx<>+24(SB)/8, $0x000000009e3779b1
GLOBL prime_avx<>(SB), RODATA|NOPTR, $32

// func accumAVX2(acc *[8]uint64, data *byte, key *byte, len uint64)
// Requires: AVX, AVX2, MMX+
TEXT ·accumAVX2(SB), NOSPLIT, $0-32
	MOVQ    acc+0(FP), AX
	MOVQ    data+8(FP), CX
	MOVQ    key+16(FP), DX
	MOVQ    key+16(FP), BX
	MOVQ    len+24(FP), SI
	VMOVDQU (AX), Y1
	VMOVDQU 32(AX), Y2
	VMOVDQU prime_avx<>+0(SB), Y0
	CMPQ    SI, $0x00000400
	JLE     accum
	VMOVDQU 32(DX), Y5
	VMOVDQU 40(DX), Y6
	VMOVDQU 48(DX), Y7
	VMOVDQU 56(DX), Y8
	VMOVDQU 64(DX), Y9

accum_large:
	VMOVDQU    (CX), Y3
	VMOVDQU    32(CX), Y4
	PREFETCHT0 1024(CX)
	VPXOR      (DX), Y3, Y10
	VPXOR      Y5, Y4, Y12
	VPSHUFD    $0x31, Y10, Y11
	VPSHUFD    $0x31, Y12, Y13
	VPMULUDQ   Y10, Y11, Y10
	VPMULUDQ   Y12, Y13, Y12
	VPSHUFD    $0x4e, Y3, Y3
	VPSHUFD    $0x4e, Y4, Y4
	VPADDQ     Y1, Y10, Y1
	VPADDQ     Y2, Y12, Y2
	VMOVDQU    64(CX), Y10
	VMOVDQU    96(CX), Y13
	PREFETCHT0 1088(CX)
	VPXOR      8(DX), Y10, Y11
	VPXOR      Y6, Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    128(CX), Y10
	VMOVDQU    160(CX), Y13
	PREFETCHT0 1152(CX)
	VPXOR      16(DX), Y10, Y11
	VPXOR      Y7, Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    192(CX), Y10
	VMOVDQU    224(CX), Y13
	PREFETCHT0 1216(CX)
	VPXOR      24(DX), Y10, Y11
	VPXOR      Y8, Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    256(CX), Y10
	VMOVDQU    288(CX), Y13
	PREFETCHT0 1280(CX)
	VPXOR      Y5, Y10, Y11
	VPXOR      Y9, Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    320(CX), Y10
	VMOVDQU    352(CX), Y13
	PREFETCHT0 1344(CX)
	VPXOR      Y6, Y10, Y11
	VPXOR      72(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    384(CX), Y10
	VMOVDQU    416(CX), Y13
	PREFETCHT0 1408(CX)
	VPXOR      Y7, Y10, Y11
	VPXOR      80(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    448(CX), Y10
	VMOVDQU    480(CX), Y13
	PREFETCHT0 1472(CX)
	VPXOR      Y8, Y10, Y11
	VPXOR      88(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    512(CX), Y10
	VMOVDQU    544(CX), Y13
	PREFETCHT0 1536(CX)
	VPXOR      Y9, Y10, Y11
	VPXOR      96(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    576(CX), Y10
	VMOVDQU    608(CX), Y13
	PREFETCHT0 1600(CX)
	VPXOR      72(DX), Y10, Y11
	VPXOR      104(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    640(CX), Y10
	VMOVDQU    672(CX), Y13
	PREFETCHT0 1664(CX)
	VPXOR      80(DX), Y10, Y11
	VPXOR      112(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    704(CX), Y10
	VMOVDQU    736(CX), Y13
	PREFETCHT0 1728(CX)
	VPXOR      88(DX), Y10, Y11
	VPXOR      120(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    768(CX), Y10
	VMOVDQU    800(CX), Y13
	PREFETCHT0 1792(CX)
	VPXOR      96(DX), Y10, Y11
	VPXOR      128(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    832(CX), Y10
	VMOVDQU    864(CX), Y13
	PREFETCHT0 1856(CX)
	VPXOR      104(DX), Y10, Y11
	VPXOR      136(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    896(CX), Y10
	VMOVDQU    928(CX), Y13
	PREFETCHT0 1920(CX)
	VPXOR      112(DX), Y10, Y11
	VPXOR      144(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VMOVDQU    960(CX), Y10
	VMOVDQU    992(CX), Y13
	PREFETCHT0 1984(CX)
	VPXOR      120(DX), Y10, Y11
	VPXOR      152(DX), Y13, Y14
	VPSHUFD    $0x31, Y11, Y12
	VPSHUFD    $0x31, Y14, Y15
	VPMULUDQ   Y11, Y12, Y11
	VPMULUDQ   Y14, Y15, Y14
	VPSHUFD    $0x4e, Y10, Y10
	VPSHUFD    $0x4e, Y13, Y13
	VPADDQ     Y1, Y11, Y1
	VPADDQ     Y2, Y14, Y2
	VPADDQ     Y3, Y10, Y3
	VPADDQ     Y4, Y13, Y4
	VPADDQ     Y1, Y3, Y1
	VPADDQ     Y2, Y4, Y2
	ADDQ       $0x00000400, CX
	SUBQ       $0x00000400, SI
	VPSRLQ     $0x2f, Y1, Y3
	VPXOR      Y1, Y3, Y3
	VPXOR      128(DX), Y3, Y3
	VPMULUDQ   Y0, Y3, Y1
	VPSHUFD    $0xf5, Y3, Y3
	VPMULUDQ   Y0, Y3, Y3
	VPSLLQ     $0x20, Y3, Y3
	VPADDQ     Y1, Y3, Y1
	VPSRLQ     $0x2f, Y2, Y3
	VPXOR      Y2, Y3, Y3
	VPXOR      160(DX), Y3, Y3
	VPMULUDQ   Y0, Y3, Y2
	VPSHUFD    $0xf5, Y3, Y3
	VPMULUDQ   Y0, Y3, Y3
	VPSLLQ     $0x20, Y3, Y3
	VPADDQ     Y2, Y3, Y2
	CMPQ       SI, $0x00000400
	JLE        accum
	JMP        accum_large

accum:
	CMPQ     SI, $0x40
	JLE      finalize
	VMOVDQU  (CX), Y0
	VMOVDQU  32(CX), Y5
	VPXOR    (BX), Y0, Y3
	VPXOR    32(BX), Y5, Y6
	VPSHUFD  $0x31, Y3, Y4
	VPSHUFD  $0x31, Y6, Y7
	VPMULUDQ Y3, Y4, Y3
	VPMULUDQ Y6, Y7, Y6
	VPSHUFD  $0x4e, Y0, Y0
	VPSHUFD  $0x4e, Y5, Y5
	VPADDQ   Y1, Y3, Y1
	VPADDQ   Y2, Y6, Y2
	VPADDQ   Y1, Y0, Y1
	VPADDQ   Y2, Y5, Y2
	ADDQ     $0x00000040, CX
	SUBQ     $0x00000040, SI
	ADDQ     $0x00000008, BX
	JMP      accum

finalize:
	CMPQ     SI, $0x00
	JE       return
	SUBQ     $0x40, CX
	ADDQ     SI, CX
	VMOVDQU  (CX), Y0
	VMOVDQU  32(CX), Y5
	VPXOR    121(DX), Y0, Y3
	VPXOR    153(DX), Y5, Y6
	VPSHUFD  $0x31, Y3, Y4
	VPSHUFD  $0x31, Y6, Y7
	VPMULUDQ Y3, Y4, Y3
	VPMULUDQ Y6, Y7, Y6
	VPSHUFD  $0x4e, Y0, Y0
	VPSHUFD  $0x4e, Y5, Y5
	VPADDQ   Y1, Y3, Y1
	VPADDQ   Y2, Y6, Y2
	VPADDQ   Y1, Y0, Y1
	VPADDQ   Y2, Y5, Y2

return:
	VMOVDQU Y1, (AX)
	VMOVDQU Y2, 32(AX)
	VZEROUPPER
	RET

// func accumBlockAVX2(acc *[8]uint64, data *byte, key *byte)
// Requires: AVX, AVX2
TEXT ·accumBlockAVX2(SB), NOSPLIT, $0-24
	MOVQ     acc+0(FP), AX
	MOVQ     data+8(FP), CX
	MOVQ     key+16(FP), DX
	VMOVDQU  (AX), Y1
	VMOVDQU  32(AX), Y2
	VMOVDQU  prime_avx<>+0(SB), Y0
	VMOVDQU  (CX), Y3
	VMOVDQU  32(CX), Y4
	VPXOR    (DX), Y3, Y5
	VPXOR    32(DX), Y4, Y7
	VPSHUFD  $0x31, Y5, Y6
	VPSHUFD  $0x31, Y7, Y8
	VPMULUDQ Y5, Y6, Y5
	VPMULUDQ Y7, Y8, Y7
	VPSHUFD  $0x4e, Y3, Y3
	VPSHUFD  $0x4e, Y4, Y4
	VPADDQ   Y1, Y5, Y1
	VPADDQ   Y2, Y7, Y2
	VMOVDQU  64(CX), Y5
	VMOVDQU  96(CX), Y8
	VPXOR    8(DX), Y5, Y6
	VPXOR    40(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  128(CX), Y5
	VMOVDQU  160(CX), Y8
	VPXOR    16(DX), Y5, Y6
	VPXOR    48(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  192(CX), Y5
	VMOVDQU  224(CX), Y8
	VPXOR    24(DX), Y5, Y6
	VPXOR    56(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  256(CX), Y5
	VMOVDQU  288(CX), Y8
	VPXOR    32(DX), Y5, Y6
	VPXOR    64(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  320(CX), Y5
	VMOVDQU  352(CX), Y8
	VPXOR    40(DX), Y5, Y6
	VPXOR    72(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  384(CX), Y5
	VMOVDQU  416(CX), Y8
	VPXOR    48(DX), Y5, Y6
	VPXOR    80(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  448(CX), Y5
	VMOVDQU  480(CX), Y8
	VPXOR    56(DX), Y5, Y6
	VPXOR    88(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  512(CX), Y5
	VMOVDQU  544(CX), Y8
	VPXOR    64(DX), Y5, Y6
	VPXOR    96(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  576(CX), Y5
	VMOVDQU  608(CX), Y8
	VPXOR    72(DX), Y5, Y6
	VPXOR    104(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  640(CX), Y5
	VMOVDQU  672(CX), Y8
	VPXOR    80(DX), Y5, Y6
	VPXOR    112(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  704(CX), Y5
	VMOVDQU  736(CX), Y8
	VPXOR    88(DX), Y5, Y6
	VPXOR    120(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  768(CX), Y5
	VMOVDQU  800(CX), Y8
	VPXOR    96(DX), Y5, Y6
	VPXOR    128(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  832(CX), Y5
	VMOVDQU  864(CX), Y8
	VPXOR    104(DX), Y5, Y6
	VPXOR    136(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  896(CX), Y5
	VMOVDQU  928(CX), Y8
	VPXOR    112(DX), Y5, Y6
	VPXOR    144(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VMOVDQU  960(CX), Y5
	VMOVDQU  992(CX), Y8
	VPXOR    120(DX), Y5, Y6
	VPXOR    152(DX), Y8, Y9
	VPSHUFD  $0x31, Y6, Y7
	VPSHUFD  $0x31, Y9, Y10
	VPMULUDQ Y6, Y7, Y6
	VPMULUDQ Y9, Y10, Y9
	VPSHUFD  $0x4e, Y5, Y5
	VPSHUFD  $0x4e, Y8, Y8
	VPADDQ   Y1, Y6, Y1
	VPADDQ   Y2, Y9, Y2
	VPADDQ   Y3, Y5, Y3
	VPADDQ   Y4, Y8, Y4
	VPADDQ   Y1, Y3, Y1
	VPADDQ   Y2, Y4, Y2
	VPSRLQ   $0x2f, Y1, Y3
	VPXOR    Y1, Y3, Y3
	VPXOR    128(DX), Y3, Y3
	VPMULUDQ Y0, Y3, Y1
	VPSHUFD  $0xf5, Y3, Y3
	VPMULUDQ Y0, Y3, Y3
	VPSLLQ   $0x20, Y3, Y3
	VPADDQ   Y1, Y3, Y1
	VPSRLQ   $0x2f, Y2, Y3
	VPXOR    Y2, Y3, Y3
	VPXOR    160(DX), Y3, Y3
	VPMULUDQ Y0, Y3, Y2
	VPSHUFD  $0xf5, Y3, Y3
	VPMULUDQ Y0, Y3, Y3
	VPSLLQ   $0x20, Y3, Y3
	VPADDQ   Y2, Y3, Y2
	VMOVDQU  Y1, (AX)
	VMOVDQU  Y2, 32(AX)
	VZEROUPPER
	RET
