// Code generated by command: go run gen.go -out ../galois_gen_amd64.s -stubs ../galois_gen_amd64.go -pkg=reedsolomon. DO NOT EDIT.

//go:build !appengine && !noasm && !nogen && !nopshufb && gc

#include "textflag.h"

// func _dummy_()
TEXT ·_dummy_(SB), $0
#ifdef GOAMD64_v4
#define XOR3WAY(ignore, a, b, dst) \
	VPTERNLOGD $0x96, a, b, dst

#else
#define XOR3WAY(ignore, a, b, dst) \
	VPXOR a, dst, dst \
	VPXOR b, dst, dst

#endif
	RET

// sSE2XorSlice will XOR in with out and store in out.
// Processes 16 bytes/loop.

// func sSE2XorSlice(in []byte, out []byte)
// Requires: SSE2
TEXT ·sSE2XorSlice(SB), $0-48
	MOVQ in_base+0(FP), AX
	MOVQ out_base+24(FP), CX
	MOVQ in_len+8(FP), DX
	SHRQ $0x04, DX
	JZ   end

loop:
	MOVOU (AX), X0
	MOVOU (CX), X1
	PXOR  X0, X1
	MOVOU X1, (CX)
	ADDQ  $0x10, AX
	ADDQ  $0x10, CX
	DECQ  DX
	JNZ   loop

end:
	RET

// sSE2XorSlice_64 will XOR in with out and store in out.
// Processes 64 bytes/loop.

// func sSE2XorSlice_64(in []byte, out []byte)
// Requires: SSE2
TEXT ·sSE2XorSlice_64(SB), $0-48
	MOVQ in_base+0(FP), AX
	MOVQ out_base+24(FP), CX
	MOVQ in_len+8(FP), DX
	SHRQ $0x06, DX
	JZ   end

loop:
	MOVOU (AX), X0
	MOVOU 16(AX), X2
	MOVOU 32(AX), X4
	MOVOU 48(AX), X6
	MOVOU (CX), X1
	MOVOU 16(CX), X3
	MOVOU 32(CX), X5
	MOVOU 48(CX), X7
	PXOR  X0, X1
	PXOR  X2, X3
	PXOR  X4, X5
	PXOR  X6, X7
	MOVOU X1, (CX)
	MOVOU X3, 16(CX)
	MOVOU X5, 32(CX)
	MOVOU X7, 48(CX)
	ADDQ  $0x40, AX
	ADDQ  $0x40, CX
	DECQ  DX
	JNZ   loop

end:
	RET

// avx2XorSlice_64 will XOR in with out and store in out.
// Processes 64 bytes/loop.

// func avx2XorSlice_64(in []byte, out []byte)
// Requires: AVX, AVX2
TEXT ·avx2XorSlice_64(SB), $0-48
	MOVQ in_base+0(FP), AX
	MOVQ out_base+24(FP), CX
	MOVQ in_len+8(FP), DX
	SHRQ $0x06, DX
	JZ   end

loop:
	VMOVDQU (AX), Y0
	VMOVDQU 32(AX), Y2
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y3
	VPXOR   Y0, Y1, Y1
	VPXOR   Y2, Y3, Y3
	VMOVDQU Y1, (CX)
	VMOVDQU Y3, 32(CX)
	ADDQ    $0x40, AX
	ADDQ    $0x40, CX
	DECQ    DX
	JNZ     loop

end:
	VZEROUPPER
	RET

// func mulAvxTwo_1x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ    n+80(FP), AX
	MOVQ    matrix_base+0(FP), CX
	SHRQ    $0x06, AX
	TESTQ   AX, AX
	JZ      mulAvxTwo_1x1_64_end
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1
	MOVQ    in_base+24(FP), CX
	MOVQ    (CX), CX
	MOVQ    out_base+48(FP), DX
	MOVQ    (DX), DX
	MOVQ    start+72(FP), BX

	// Add start offset to output
	ADDQ BX, DX

	// Add start offset to input
	ADDQ         BX, CX
	MOVQ         $0x0000000f, BX
	MOVQ         BX, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_1x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (CX), Y2
	VMOVDQU 32(CX), Y3
	ADDQ    $0x40, CX
	VPSRLQ  $0x04, Y2, Y6
	VPSRLQ  $0x04, Y3, Y5
	VPAND   Y4, Y2, Y2
	VPAND   Y4, Y3, Y3
	VPAND   Y4, Y6, Y6
	VPAND   Y4, Y5, Y5
	VPSHUFB Y2, Y0, Y2
	VPSHUFB Y3, Y0, Y3
	VPSHUFB Y6, Y1, Y6
	VPSHUFB Y5, Y1, Y5
	VPXOR   Y2, Y6, Y2
	VPXOR   Y3, Y5, Y3

	// Store 1 outputs
	VMOVDQU Y2, (DX)
	VMOVDQU Y3, 32(DX)
	ADDQ    $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x1_64_loop
	VZEROUPPER

mulAvxTwo_1x1_64_end:
	RET

// func mulGFNI_1x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 4 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x1_64_end
	VBROADCASTF32X2 (CX), Z0
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), DX
	MOVQ            start+72(FP), BX

	// Add start offset to output
	ADDQ BX, DX

	// Add start offset to input
	ADDQ BX, CX

mulGFNI_1x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (CX), Z1
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z1, Z1

	// Store 1 outputs
	VMOVDQU64 Z1, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x1_64_loop
	VZEROUPPER

mulGFNI_1x1_64_end:
	RET

// func mulAvxGFNI_1x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 4 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x1_end
	VBROADCASTSD (CX), Y0
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), DX
	MOVQ         start+72(FP), BX

	// Add start offset to output
	ADDQ BX, DX

	// Add start offset to input
	ADDQ BX, CX

mulAvxGFNI_1x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (CX), Y1
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y1, Y1

	// Store 1 outputs
	VMOVDQU Y1, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x1_loop
	VZEROUPPER

mulAvxGFNI_1x1_end:
	RET

// func mulGFNI_1x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 4 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), DX
	MOVQ            start+72(FP), BX

	// Add start offset to output
	ADDQ BX, DX

	// Add start offset to input
	ADDQ BX, CX

mulGFNI_1x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (DX), Z1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (CX), Z2
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z2, Z2
	VXORPD         Z1, Z2, Z1

	// Store 1 outputs
	VMOVDQU64 Z1, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x1_64Xor_loop
	VZEROUPPER

mulGFNI_1x1_64Xor_end:
	RET

// func mulAvxGFNI_1x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 4 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x1Xor_end
	VBROADCASTSD (CX), Y0
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), DX
	MOVQ         start+72(FP), BX

	// Add start offset to output
	ADDQ BX, DX

	// Add start offset to input
	ADDQ BX, CX

mulAvxGFNI_1x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (DX), Y1

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (CX), Y2
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y2, Y2
	VXORPD         Y1, Y2, Y1

	// Store 1 outputs
	VMOVDQU Y1, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x1Xor_loop
	VZEROUPPER

mulAvxGFNI_1x1Xor_end:
	RET

// func mulAvxTwo_1x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ    n+80(FP), AX
	MOVQ    matrix_base+0(FP), CX
	SHRQ    $0x06, AX
	TESTQ   AX, AX
	JZ      mulAvxTwo_1x1_64Xor_end
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1
	MOVQ    in_base+24(FP), CX
	MOVQ    (CX), CX
	MOVQ    out_base+48(FP), DX
	MOVQ    (DX), DX
	MOVQ    start+72(FP), BX

	// Add start offset to output
	ADDQ BX, DX

	// Add start offset to input
	ADDQ         BX, CX
	MOVQ         $0x0000000f, BX
	MOVQ         BX, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_1x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (DX), Y2
	VMOVDQU 32(DX), Y3

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y7
	ADDQ    $0x40, CX
	VPSRLQ  $0x04, Y5, Y6
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y5, Y5
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y6, Y6
	VPAND   Y4, Y8, Y8
	VPSHUFB Y5, Y0, Y5
	VPSHUFB Y7, Y0, Y7
	VPSHUFB Y6, Y1, Y6
	VPSHUFB Y8, Y1, Y8
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 1 outputs
	VMOVDQU Y2, (DX)
	VMOVDQU Y3, 32(DX)
	ADDQ    $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_1x1_64Xor_end:
	RET

// func mulAvxTwo_1x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), BX
	MOVQ  start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI
	ADDQ DI, BX

	// Add start offset to input
	ADDQ         DI, DX
	MOVQ         $0x0000000f, DI
	MOVQ         DI, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_1x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (DX), Y7
	VMOVDQU 32(DX), Y9
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y7, Y8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y8, Y8
	VPAND   Y4, Y10, Y10
	VMOVDQU (CX), Y2
	VMOVDQU 32(CX), Y6
	VPSHUFB Y9, Y2, Y3
	VPSHUFB Y7, Y2, Y2
	VPSHUFB Y10, Y6, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y2, Y6, Y0
	VPXOR   Y3, Y5, Y1
	VMOVDQU 64(CX), Y2
	VMOVDQU 96(CX), Y6
	VPSHUFB Y9, Y2, Y3
	VPSHUFB Y7, Y2, Y2
	VPSHUFB Y10, Y6, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y2, Y6, Y2
	VPXOR   Y3, Y5, Y3

	// Store 2 outputs
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (BX)
	VMOVDQU Y3, 32(BX)
	ADDQ    $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x2_64_loop
	VZEROUPPER

mulAvxTwo_1x2_64_end:
	RET

// func mulGFNI_1x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), DX
	MOVQ            start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX
	ADDQ SI, DX

	// Add start offset to input
	ADDQ SI, CX

mulGFNI_1x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (CX), Z3
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z2
	VGF2P8AFFINEQB $0x00, Z1, Z3, Z3

	// Store 2 outputs
	VMOVDQU64 Z2, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z3, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x2_64_loop
	VZEROUPPER

mulGFNI_1x2_64_end:
	RET

// func mulAvxGFNI_1x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x2(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX
	ADDQ SI, DX

	// Add start offset to input
	ADDQ SI, CX

mulAvxGFNI_1x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (CX), Y3
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y3, Y2
	VGF2P8AFFINEQB $0x00, Y1, Y3, Y3

	// Store 2 outputs
	VMOVDQU Y2, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y3, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x2_loop
	VZEROUPPER

mulAvxGFNI_1x2_end:
	RET

// func mulGFNI_1x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), DX
	MOVQ            start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX
	ADDQ SI, DX

	// Add start offset to input
	ADDQ SI, CX

mulGFNI_1x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (BX), Z2
	VMOVDQU64 (DX), Z3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (CX), Z4
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
	VXORPD         Z2, Z5, Z2
	VGF2P8AFFINEQB $0x00, Z1, Z4, Z5
	VXORPD         Z3, Z5, Z3

	// Store 2 outputs
	VMOVDQU64 Z2, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z3, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x2_64Xor_loop
	VZEROUPPER

mulGFNI_1x2_64Xor_end:
	RET

// func mulAvxGFNI_1x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x2Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX
	ADDQ SI, DX

	// Add start offset to input
	ADDQ SI, CX

mulAvxGFNI_1x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (BX), Y2
	VMOVDQU (DX), Y3

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (CX), Y4
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y4, Y5
	VXORPD         Y2, Y5, Y2
	VGF2P8AFFINEQB $0x00, Y1, Y4, Y5
	VXORPD         Y3, Y5, Y3

	// Store 2 outputs
	VMOVDQU Y2, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y3, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x2Xor_loop
	VZEROUPPER

mulAvxGFNI_1x2Xor_end:
	RET

// func mulAvxTwo_1x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), BX
	MOVQ  start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI
	ADDQ DI, BX

	// Add start offset to input
	ADDQ         DI, DX
	MOVQ         $0x0000000f, DI
	MOVQ         DI, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_1x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y1
	VMOVDQU (BX), Y2
	VMOVDQU 32(BX), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (BX)
	VMOVDQU Y3, 32(BX)
	ADDQ    $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_1x2_64Xor_end:
	RET

// func mulAvxTwo_1x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), BX
	MOVQ  start+72(FP), R8

	// Add start offset to output
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, BX

	// Add start offset to input
	ADDQ         R8, DX
	MOVQ         $0x0000000f, R8
	MOVQ         R8, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_1x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y10, Y10
	VPAND   Y6, Y12, Y12
	VMOVDQU (CX), Y4
	VMOVDQU 32(CX), Y8
	VPSHUFB Y11, Y4, Y5
	VPSHUFB Y9, Y4, Y4
	VPSHUFB Y12, Y8, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y4, Y8, Y0
	VPXOR   Y5, Y7, Y1
	VMOVDQU 64(CX), Y4
	VMOVDQU 96(CX), Y8
	VPSHUFB Y11, Y4, Y5
	VPSHUFB Y9, Y4, Y4
	VPSHUFB Y12, Y8, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y4, Y8, Y2
	VPXOR   Y5, Y7, Y3
	VMOVDQU 128(CX), Y4
	VMOVDQU 160(CX), Y8
	VPSHUFB Y11, Y4, Y5
	VPSHUFB Y9, Y4, Y4
	VPSHUFB Y12, Y8, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y4, Y8, Y4
	VPXOR   Y5, Y7, Y5

	// Store 3 outputs
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y4, (BX)
	VMOVDQU Y5, 32(BX)
	ADDQ    $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x3_64_loop
	VZEROUPPER

mulAvxTwo_1x3_64_end:
	RET

// func mulGFNI_1x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DX
	MOVQ            start+72(FP), DI

	// Add start offset to output
	ADDQ DI, BX
	ADDQ DI, SI
	ADDQ DI, DX

	// Add start offset to input
	ADDQ DI, CX

mulGFNI_1x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (CX), Z5
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z5, Z3
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z4
	VGF2P8AFFINEQB $0x00, Z2, Z5, Z5

	// Store 3 outputs
	VMOVDQU64 Z3, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z4, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z5, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x3_64_loop
	VZEROUPPER

mulGFNI_1x3_64_end:
	RET

// func mulAvxGFNI_1x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x3(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         start+72(FP), DI

	// Add start offset to output
	ADDQ DI, BX
	ADDQ DI, SI
	ADDQ DI, DX

	// Add start offset to input
	ADDQ DI, CX

mulAvxGFNI_1x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (CX), Y5
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y5, Y3
	VGF2P8AFFINEQB $0x00, Y1, Y5, Y4
	VGF2P8AFFINEQB $0x00, Y2, Y5, Y5

	// Store 3 outputs
	VMOVDQU Y3, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y5, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x3_loop
	VZEROUPPER

mulAvxGFNI_1x3_end:
	RET

// func mulGFNI_1x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DX
	MOVQ            start+72(FP), DI

	// Add start offset to output
	ADDQ DI, BX
	ADDQ DI, SI
	ADDQ DI, DX

	// Add start offset to input
	ADDQ DI, CX

mulGFNI_1x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (BX), Z3
	VMOVDQU64 (SI), Z4
	VMOVDQU64 (DX), Z5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (CX), Z6
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z6, Z7
	VXORPD         Z3, Z7, Z3
	VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
	VXORPD         Z4, Z7, Z4
	VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
	VXORPD         Z5, Z7, Z5

	// Store 3 outputs
	VMOVDQU64 Z3, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z4, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z5, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x3_64Xor_loop
	VZEROUPPER

mulGFNI_1x3_64Xor_end:
	RET

// func mulAvxGFNI_1x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x3Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         start+72(FP), DI

	// Add start offset to output
	ADDQ DI, BX
	ADDQ DI, SI
	ADDQ DI, DX

	// Add start offset to input
	ADDQ DI, CX

mulAvxGFNI_1x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (BX), Y3
	VMOVDQU (SI), Y4
	VMOVDQU (DX), Y5

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (CX), Y6
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y6, Y7
	VXORPD         Y3, Y7, Y3
	VGF2P8AFFINEQB $0x00, Y1, Y6, Y7
	VXORPD         Y4, Y7, Y4
	VGF2P8AFFINEQB $0x00, Y2, Y6, Y7
	VXORPD         Y5, Y7, Y5

	// Store 3 outputs
	VMOVDQU Y3, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y5, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x3Xor_loop
	VZEROUPPER

mulAvxGFNI_1x3Xor_end:
	RET

// func mulAvxTwo_1x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), BX
	MOVQ  start+72(FP), R8

	// Add start offset to output
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, BX

	// Add start offset to input
	ADDQ         R8, DX
	MOVQ         $0x0000000f, R8
	MOVQ         R8, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_1x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y1
	VMOVDQU (DI), Y2
	VMOVDQU 32(DI), Y3
	VMOVDQU (BX), Y4
	VMOVDQU 32(BX), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y4, (BX)
	VMOVDQU Y5, 32(BX)
	ADDQ    $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_1x3_64Xor_end:
	RET

// func mulAvxTwo_1x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x4(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), BX
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, BX

	// Add start offset to input
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_1x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (DX), Y6
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y6, Y7
	VPAND   Y4, Y6, Y6
	VPAND   Y4, Y7, Y7
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y7, Y5, Y5
	VPXOR   Y3, Y5, Y0
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y7, Y5, Y5
	VPXOR   Y3, Y5, Y1
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y7, Y5, Y5
	VPXOR   Y3, Y5, Y2
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y7, Y5, Y5
	VPXOR   Y3, Y5, Y3

	// Store 4 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x4_loop
	VZEROUPPER

mulAvxTwo_1x4_end:
	RET

// func mulGFNI_1x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x4_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, DX

	// Add start offset to input
	ADDQ R8, CX

mulGFNI_1x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (CX), Z7
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z7, Z4
	VGF2P8AFFINEQB $0x00, Z1, Z7, Z5
	VGF2P8AFFINEQB $0x00, Z2, Z7, Z6
	VGF2P8AFFINEQB $0x00, Z3, Z7, Z7

	// Store 4 outputs
	VMOVDQU64 Z4, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z5, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z6, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z7, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x4_64_loop
	VZEROUPPER

mulGFNI_1x4_64_end:
	RET

// func mulAvxGFNI_1x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x4(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, DX

	// Add start offset to input
	ADDQ R8, CX

mulAvxGFNI_1x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (CX), Y7
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y7, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y7, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y7, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y7, Y7

	// Store 4 outputs
	VMOVDQU Y4, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y5, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x4_loop
	VZEROUPPER

mulAvxGFNI_1x4_end:
	RET

// func mulGFNI_1x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x4_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, DX

	// Add start offset to input
	ADDQ R8, CX

mulGFNI_1x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (BX), Z4
	VMOVDQU64 (SI), Z5
	VMOVDQU64 (DI), Z6
	VMOVDQU64 (DX), Z7

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (CX), Z8
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z8, Z9
	VXORPD         Z4, Z9, Z4
	VGF2P8AFFINEQB $0x00, Z1, Z8, Z9
	VXORPD         Z5, Z9, Z5
	VGF2P8AFFINEQB $0x00, Z2, Z8, Z9
	VXORPD         Z6, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z3, Z8, Z9
	VXORPD         Z7, Z9, Z7

	// Store 4 outputs
	VMOVDQU64 Z4, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z5, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z6, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z7, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x4_64Xor_loop
	VZEROUPPER

mulGFNI_1x4_64Xor_end:
	RET

// func mulAvxGFNI_1x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x4Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, DX

	// Add start offset to input
	ADDQ R8, CX

mulAvxGFNI_1x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (BX), Y4
	VMOVDQU (SI), Y5
	VMOVDQU (DI), Y6
	VMOVDQU (DX), Y7

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (CX), Y8
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y8, Y9
	VXORPD         Y4, Y9, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y8, Y9
	VXORPD         Y5, Y9, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y8, Y9
	VXORPD         Y6, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y8, Y9
	VXORPD         Y7, Y9, Y7

	// Store 4 outputs
	VMOVDQU Y4, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y5, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x4Xor_loop
	VZEROUPPER

mulAvxGFNI_1x4Xor_end:
	RET

// func mulAvxTwo_1x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x4Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), BX
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, BX

	// Add start offset to input
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_1x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (SI), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (DI), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R8), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (BX), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x4Xor_loop
	VZEROUPPER

mulAvxTwo_1x4Xor_end:
	RET

// func mulAvxTwo_1x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x5(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), BX
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, BX

	// Add start offset to input
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_1x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y5, Y7, Y7
	VPAND   Y5, Y8, Y8
	VMOVDQU (CX), Y4
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y4, Y4
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y4, Y6, Y0
	VMOVDQU 64(CX), Y4
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y4, Y4
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y4, Y6, Y1
	VMOVDQU 128(CX), Y4
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y4, Y4
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y4, Y6, Y2
	VMOVDQU 192(CX), Y4
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y4, Y4
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y4, Y6, Y3
	VMOVDQU 256(CX), Y4
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y4, Y4
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y4, Y6, Y4

	// Store 5 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x5_loop
	VZEROUPPER

mulAvxTwo_1x5_end:
	RET

// func mulGFNI_1x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x5_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, DX

	// Add start offset to input
	ADDQ R9, CX

mulGFNI_1x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (CX), Z9
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z9, Z5
	VGF2P8AFFINEQB $0x00, Z1, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z2, Z9, Z7
	VGF2P8AFFINEQB $0x00, Z3, Z9, Z8
	VGF2P8AFFINEQB $0x00, Z4, Z9, Z9

	// Store 5 outputs
	VMOVDQU64 Z5, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z6, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z7, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z8, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z9, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x5_64_loop
	VZEROUPPER

mulGFNI_1x5_64_end:
	RET

// func mulAvxGFNI_1x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x5(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, DX

	// Add start offset to input
	ADDQ R9, CX

mulAvxGFNI_1x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (CX), Y9
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y9, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y9, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y9, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y9, Y9

	// Store 5 outputs
	VMOVDQU Y5, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x5_loop
	VZEROUPPER

mulAvxGFNI_1x5_end:
	RET

// func mulGFNI_1x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x5_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, DX

	// Add start offset to input
	ADDQ R9, CX

mulGFNI_1x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (BX), Z5
	VMOVDQU64 (SI), Z6
	VMOVDQU64 (DI), Z7
	VMOVDQU64 (R8), Z8
	VMOVDQU64 (DX), Z9

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (CX), Z10
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z10, Z11
	VXORPD         Z5, Z11, Z5
	VGF2P8AFFINEQB $0x00, Z1, Z10, Z11
	VXORPD         Z6, Z11, Z6
	VGF2P8AFFINEQB $0x00, Z2, Z10, Z11
	VXORPD         Z7, Z11, Z7
	VGF2P8AFFINEQB $0x00, Z3, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z4, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Store 5 outputs
	VMOVDQU64 Z5, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z6, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z7, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z8, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z9, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x5_64Xor_loop
	VZEROUPPER

mulGFNI_1x5_64Xor_end:
	RET

// func mulAvxGFNI_1x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x5Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, DX

	// Add start offset to input
	ADDQ R9, CX

mulAvxGFNI_1x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (BX), Y5
	VMOVDQU (SI), Y6
	VMOVDQU (DI), Y7
	VMOVDQU (R8), Y8
	VMOVDQU (DX), Y9

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (CX), Y10
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y10, Y11
	VXORPD         Y5, Y11, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y10, Y11
	VXORPD         Y6, Y11, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y10, Y11
	VXORPD         Y7, Y11, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Store 5 outputs
	VMOVDQU Y5, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x5Xor_loop
	VZEROUPPER

mulAvxGFNI_1x5Xor_end:
	RET

// func mulAvxTwo_1x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x5Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), BX
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, BX

	// Add start offset to input
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_1x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (SI), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (DI), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R8), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R9), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (BX), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x5Xor_loop
	VZEROUPPER

mulAvxTwo_1x5Xor_end:
	RET

// func mulAvxTwo_1x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x6(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), BX
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, BX

	// Add start offset to input
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_1x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y6, Y8, Y8
	VPAND   Y6, Y9, Y9
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y5, Y5
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y5, Y7, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y5, Y5
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y5, Y7, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y5, Y5
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y5, Y7, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y5, Y5
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y5, Y7, Y3
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y5, Y5
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y5, Y7, Y4
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y5, Y5
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y5, Y7, Y5

	// Store 6 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x6_loop
	VZEROUPPER

mulAvxTwo_1x6_end:
	RET

// func mulGFNI_1x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x6_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DX

	// Add start offset to input
	ADDQ R10, CX

mulGFNI_1x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (CX), Z11
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z11, Z6
	VGF2P8AFFINEQB $0x00, Z1, Z11, Z7
	VGF2P8AFFINEQB $0x00, Z2, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z3, Z11, Z9
	VGF2P8AFFINEQB $0x00, Z4, Z11, Z10
	VGF2P8AFFINEQB $0x00, Z5, Z11, Z11

	// Store 6 outputs
	VMOVDQU64 Z6, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z7, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z8, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z9, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z10, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z11, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x6_64_loop
	VZEROUPPER

mulGFNI_1x6_64_end:
	RET

// func mulAvxGFNI_1x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x6(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         start+72(FP), R10

	// Add start offset to output
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DX

	// Add start offset to input
	ADDQ R10, CX

mulAvxGFNI_1x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (CX), Y11
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y11, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y11, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y11, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y11, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y11, Y11

	// Store 6 outputs
	VMOVDQU Y6, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y7, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y9, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x6_loop
	VZEROUPPER

mulAvxGFNI_1x6_end:
	RET

// func mulGFNI_1x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x6_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DX

	// Add start offset to input
	ADDQ R10, CX

mulGFNI_1x6_64Xor_loop:
	// Load 6 outputs
	VMOVDQU64 (BX), Z6
	VMOVDQU64 (SI), Z7
	VMOVDQU64 (DI), Z8
	VMOVDQU64 (R8), Z9
	VMOVDQU64 (R9), Z10
	VMOVDQU64 (DX), Z11

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (CX), Z12
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
	VXORPD         Z6, Z13, Z6
	VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
	VXORPD         Z7, Z13, Z7
	VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
	VXORPD         Z8, Z13, Z8
	VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Store 6 outputs
	VMOVDQU64 Z6, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z7, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z8, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z9, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z10, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z11, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x6_64Xor_loop
	VZEROUPPER

mulGFNI_1x6_64Xor_end:
	RET

// func mulAvxGFNI_1x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x6Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         start+72(FP), R10

	// Add start offset to output
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DX

	// Add start offset to input
	ADDQ R10, CX

mulAvxGFNI_1x6Xor_loop:
	// Load 6 outputs
	VMOVDQU (BX), Y6
	VMOVDQU (SI), Y7
	VMOVDQU (DI), Y8
	VMOVDQU (R8), Y9
	VMOVDQU (R9), Y10
	VMOVDQU (DX), Y11

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (CX), Y12
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
	VXORPD         Y6, Y13, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
	VXORPD         Y7, Y13, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
	VXORPD         Y8, Y13, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Store 6 outputs
	VMOVDQU Y6, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y7, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y9, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x6Xor_loop
	VZEROUPPER

mulAvxGFNI_1x6Xor_end:
	RET

// func mulAvxTwo_1x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x6Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), BX
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, BX

	// Add start offset to input
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_1x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (SI), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU (DI), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU (R8), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU (R9), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU (R10), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU (BX), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x6Xor_loop
	VZEROUPPER

mulAvxTwo_1x6Xor_end:
	RET

// func mulAvxTwo_1x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x7(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), BX
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, BX

	// Add start offset to input
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_1x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y7, Y9, Y9
	VPAND   Y7, Y10, Y10
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y6, Y6
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y6, Y8, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y6, Y6
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y6, Y8, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y6, Y6
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y6, Y8, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y6, Y6
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y6, Y8, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y6, Y6
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y6, Y8, Y4
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y6, Y6
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y6, Y8, Y5
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y6, Y6
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y6, Y8, Y6

	// Store 7 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x7_loop
	VZEROUPPER

mulAvxTwo_1x7_end:
	RET

// func mulGFNI_1x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x7_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DX

	// Add start offset to input
	ADDQ R11, CX

mulGFNI_1x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (CX), Z13
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z13, Z7
	VGF2P8AFFINEQB $0x00, Z1, Z13, Z8
	VGF2P8AFFINEQB $0x00, Z2, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z3, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z4, Z13, Z11
	VGF2P8AFFINEQB $0x00, Z5, Z13, Z12
	VGF2P8AFFINEQB $0x00, Z6, Z13, Z13

	// Store 7 outputs
	VMOVDQU64 Z7, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z8, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z10, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z11, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z12, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z13, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x7_64_loop
	VZEROUPPER

mulGFNI_1x7_64_end:
	RET

// func mulAvxGFNI_1x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x7(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DX

	// Add start offset to input
	ADDQ R11, CX

mulAvxGFNI_1x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (CX), Y13
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y13, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y13, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y13, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y13, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y13, Y13

	// Store 7 outputs
	VMOVDQU Y7, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y8, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x7_loop
	VZEROUPPER

mulAvxGFNI_1x7_end:
	RET

// func mulGFNI_1x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x7_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DX

	// Add start offset to input
	ADDQ R11, CX

mulGFNI_1x7_64Xor_loop:
	// Load 7 outputs
	VMOVDQU64 (BX), Z7
	VMOVDQU64 (SI), Z8
	VMOVDQU64 (DI), Z9
	VMOVDQU64 (R8), Z10
	VMOVDQU64 (R9), Z11
	VMOVDQU64 (R10), Z12
	VMOVDQU64 (DX), Z13

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (CX), Z14
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z14, Z15
	VXORPD         Z7, Z15, Z7
	VGF2P8AFFINEQB $0x00, Z1, Z14, Z15
	VXORPD         Z8, Z15, Z8
	VGF2P8AFFINEQB $0x00, Z2, Z14, Z15
	VXORPD         Z9, Z15, Z9
	VGF2P8AFFINEQB $0x00, Z3, Z14, Z15
	VXORPD         Z10, Z15, Z10
	VGF2P8AFFINEQB $0x00, Z4, Z14, Z15
	VXORPD         Z11, Z15, Z11
	VGF2P8AFFINEQB $0x00, Z5, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z6, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Store 7 outputs
	VMOVDQU64 Z7, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z8, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z10, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z11, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z12, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z13, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x7_64Xor_loop
	VZEROUPPER

mulGFNI_1x7_64Xor_end:
	RET

// func mulAvxGFNI_1x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x7Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), CX
	MOVQ         out_base+48(FP), DX
	MOVQ         out_base+48(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DX

	// Add start offset to input
	ADDQ R11, CX

mulAvxGFNI_1x7Xor_loop:
	// Load 7 outputs
	VMOVDQU (BX), Y7
	VMOVDQU (SI), Y8
	VMOVDQU (DI), Y9
	VMOVDQU (R8), Y10
	VMOVDQU (R9), Y11
	VMOVDQU (R10), Y12
	VMOVDQU (DX), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (CX), Y14
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (BX)
	ADDQ    $0x20, BX
	VMOVDQU Y8, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (DX)
	ADDQ    $0x20, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x7Xor_loop
	VZEROUPPER

mulAvxGFNI_1x7Xor_end:
	RET

// func mulAvxTwo_1x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x7Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), BX
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, BX

	// Add start offset to input
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_1x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (SI), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU (DI), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU (R8), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU (R9), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU (R10), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU (R11), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU (BX), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x7Xor_loop
	VZEROUPPER

mulAvxTwo_1x7Xor_end:
	RET

// func mulAvxTwo_1x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x8(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), R12
	MOVQ  168(BX), BX
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, BX

	// Add start offset to input
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_1x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y8, Y10, Y10
	VPAND   Y8, Y11, Y11
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y5
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y6
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y7, Y7
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y7, Y9, Y7

	// Store 8 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y7, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x8_loop
	VZEROUPPER

mulAvxTwo_1x8_end:
	RET

// func mulGFNI_1x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x8_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DX

	// Add start offset to input
	ADDQ R12, CX

mulGFNI_1x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (CX), Z15
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z15, Z8
	VGF2P8AFFINEQB $0x00, Z1, Z15, Z9
	VGF2P8AFFINEQB $0x00, Z2, Z15, Z10
	VGF2P8AFFINEQB $0x00, Z3, Z15, Z11
	VGF2P8AFFINEQB $0x00, Z4, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z5, Z15, Z13
	VGF2P8AFFINEQB $0x00, Z6, Z15, Z14
	VGF2P8AFFINEQB $0x00, Z7, Z15, Z15

	// Store 8 outputs
	VMOVDQU64 Z8, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z9, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z10, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z11, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z12, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z13, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z14, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z15, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x8_64_loop
	VZEROUPPER

mulGFNI_1x8_64_end:
	RET

// func mulAvxGFNI_1x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x8(SB), $0-88
	// Loading 6 of 8 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), DX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), R9
	MOVQ         96(BX), R10
	MOVQ         120(BX), R11
	MOVQ         144(BX), R12
	MOVQ         168(BX), BX
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, BX

	// Add start offset to input
	ADDQ R13, DX

mulAvxGFNI_1x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (DX), Y13
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y13, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y13, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y13, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y13, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y13, Y12
	VBROADCASTSD   56(CX), Y14
	VGF2P8AFFINEQB $0x00, Y14, Y13, Y13

	// Store 8 outputs
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x8_loop
	VZEROUPPER

mulAvxGFNI_1x8_end:
	RET

// func mulGFNI_1x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x8_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DX

	// Add start offset to input
	ADDQ R12, CX

mulGFNI_1x8_64Xor_loop:
	// Load 8 outputs
	VMOVDQU64 (BX), Z8
	VMOVDQU64 (SI), Z9
	VMOVDQU64 (DI), Z10
	VMOVDQU64 (R8), Z11
	VMOVDQU64 (R9), Z12
	VMOVDQU64 (R10), Z13
	VMOVDQU64 (R11), Z14
	VMOVDQU64 (DX), Z15

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (CX), Z16
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z16, Z17
	VXORPD         Z8, Z17, Z8
	VGF2P8AFFINEQB $0x00, Z1, Z16, Z17
	VXORPD         Z9, Z17, Z9
	VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
	VXORPD         Z10, Z17, Z10
	VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
	VXORPD         Z11, Z17, Z11
	VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
	VXORPD         Z12, Z17, Z12
	VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
	VXORPD         Z13, Z17, Z13
	VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Store 8 outputs
	VMOVDQU64 Z8, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z9, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z10, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z11, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z12, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z13, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z14, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z15, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x8_64Xor_loop
	VZEROUPPER

mulGFNI_1x8_64Xor_end:
	RET

// func mulAvxGFNI_1x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x8Xor(SB), $0-88
	// Loading 6 of 8 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), DX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), R9
	MOVQ         96(BX), R10
	MOVQ         120(BX), R11
	MOVQ         144(BX), R12
	MOVQ         168(BX), BX
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, BX

	// Add start offset to input
	ADDQ R13, DX

mulAvxGFNI_1x8Xor_loop:
	// Load 8 outputs
	VMOVDQU (SI), Y6
	VMOVDQU (DI), Y7
	VMOVDQU (R8), Y8
	VMOVDQU (R9), Y9
	VMOVDQU (R10), Y10
	VMOVDQU (R11), Y11
	VMOVDQU (R12), Y12
	VMOVDQU (BX), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x8Xor_loop
	VZEROUPPER

mulAvxGFNI_1x8Xor_end:
	RET

// func mulAvxTwo_1x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x8Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), R12
	MOVQ  168(BX), BX
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, BX

	// Add start offset to input
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_1x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (SI), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU (DI), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU (R8), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU (R9), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU (R10), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU (R11), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU (R12), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU (BX), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y7, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x8Xor_loop
	VZEROUPPER

mulAvxTwo_1x8Xor_end:
	RET

// func mulAvxTwo_1x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x9(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), R12
	MOVQ  168(BX), R13
	MOVQ  192(BX), BX
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, BX

	// Add start offset to input
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_1x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y9, Y11, Y11
	VPAND   Y9, Y12, Y12
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y6
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y7
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y8, Y8
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y8, Y10, Y8

	// Store 9 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y7, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y8, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x9_loop
	VZEROUPPER

mulAvxTwo_1x9_end:
	RET

// func mulGFNI_1x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x9_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DX

	// Add start offset to input
	ADDQ R13, CX

mulGFNI_1x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (CX), Z17
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z17, Z9
	VGF2P8AFFINEQB $0x00, Z1, Z17, Z10
	VGF2P8AFFINEQB $0x00, Z2, Z17, Z11
	VGF2P8AFFINEQB $0x00, Z3, Z17, Z12
	VGF2P8AFFINEQB $0x00, Z4, Z17, Z13
	VGF2P8AFFINEQB $0x00, Z5, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z6, Z17, Z15
	VGF2P8AFFINEQB $0x00, Z7, Z17, Z16
	VGF2P8AFFINEQB $0x00, Z8, Z17, Z17

	// Store 9 outputs
	VMOVDQU64 Z9, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z10, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z11, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z12, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z14, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z15, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z16, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z17, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x9_64_loop
	VZEROUPPER

mulGFNI_1x9_64_end:
	RET

// func mulAvxGFNI_1x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x9(SB), $0-88
	// Loading 5 of 9 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), DX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), R9
	MOVQ         96(BX), R10
	MOVQ         120(BX), R11
	MOVQ         144(BX), R12
	MOVQ         168(BX), R13
	MOVQ         192(BX), BX
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, BX

	// Add start offset to input
	ADDQ R14, DX

mulAvxGFNI_1x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (DX), Y13
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y13, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y13, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y13, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y13, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y13, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y13, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y13, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y13, Y12
	VBROADCASTSD   64(CX), Y14
	VGF2P8AFFINEQB $0x00, Y14, Y13, Y13

	// Store 9 outputs
	VMOVDQU Y5, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x9_loop
	VZEROUPPER

mulAvxGFNI_1x9_end:
	RET

// func mulGFNI_1x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x9_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DX

	// Add start offset to input
	ADDQ R13, CX

mulGFNI_1x9_64Xor_loop:
	// Load 9 outputs
	VMOVDQU64 (BX), Z9
	VMOVDQU64 (SI), Z10
	VMOVDQU64 (DI), Z11
	VMOVDQU64 (R8), Z12
	VMOVDQU64 (R9), Z13
	VMOVDQU64 (R10), Z14
	VMOVDQU64 (R11), Z15
	VMOVDQU64 (R12), Z16
	VMOVDQU64 (DX), Z17

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (CX), Z18
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
	VXORPD         Z9, Z19, Z9
	VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
	VXORPD         Z10, Z19, Z10
	VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
	VXORPD         Z11, Z19, Z11
	VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
	VXORPD         Z12, Z19, Z12
	VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
	VXORPD         Z13, Z19, Z13
	VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
	VXORPD         Z14, Z19, Z14
	VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Store 9 outputs
	VMOVDQU64 Z9, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z10, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z11, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z12, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z14, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z15, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z16, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z17, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x9_64Xor_loop
	VZEROUPPER

mulGFNI_1x9_64Xor_end:
	RET

// func mulAvxGFNI_1x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x9Xor(SB), $0-88
	// Loading 5 of 9 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), DX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), R9
	MOVQ         96(BX), R10
	MOVQ         120(BX), R11
	MOVQ         144(BX), R12
	MOVQ         168(BX), R13
	MOVQ         192(BX), BX
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, BX

	// Add start offset to input
	ADDQ R14, DX

mulAvxGFNI_1x9Xor_loop:
	// Load 9 outputs
	VMOVDQU (SI), Y5
	VMOVDQU (DI), Y6
	VMOVDQU (R8), Y7
	VMOVDQU (R9), Y8
	VMOVDQU (R10), Y9
	VMOVDQU (R11), Y10
	VMOVDQU (R12), Y11
	VMOVDQU (R13), Y12
	VMOVDQU (BX), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	VMOVDQU Y5, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x9Xor_loop
	VZEROUPPER

mulAvxGFNI_1x9Xor_end:
	RET

// func mulAvxTwo_1x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x9Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), R12
	MOVQ  168(BX), R13
	MOVQ  192(BX), BX
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, BX

	// Add start offset to input
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_1x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (SI), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU (DI), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU (R8), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU (R9), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU (R10), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU (R11), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU (R12), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU (R13), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU (BX), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y7, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y8, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x9Xor_loop
	VZEROUPPER

mulAvxTwo_1x9Xor_end:
	RET

// func mulAvxTwo_1x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, SSE2
TEXT ·mulAvxTwo_1x10(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), R12
	MOVQ  168(BX), R13
	MOVQ  192(BX), R14
	MOVQ  216(BX), BX
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, BX

	// Add start offset to input
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_1x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y10, Y12, Y12
	VPAND   Y10, Y13, Y13
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y7
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y8
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y9, Y9
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y9, Y11, Y9

	// Store 10 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y7, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y8, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y9, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x10_loop
	VZEROUPPER

mulAvxTwo_1x10_end:
	RET

// func mulGFNI_1x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x10_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, DX

	// Add start offset to input
	ADDQ R14, CX

mulGFNI_1x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (CX), Z19
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z19, Z10
	VGF2P8AFFINEQB $0x00, Z1, Z19, Z11
	VGF2P8AFFINEQB $0x00, Z2, Z19, Z12
	VGF2P8AFFINEQB $0x00, Z3, Z19, Z13
	VGF2P8AFFINEQB $0x00, Z4, Z19, Z14
	VGF2P8AFFINEQB $0x00, Z5, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z6, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z7, Z19, Z17
	VGF2P8AFFINEQB $0x00, Z8, Z19, Z18
	VGF2P8AFFINEQB $0x00, Z9, Z19, Z19

	// Store 10 outputs
	VMOVDQU64 Z10, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z11, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z12, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z13, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z14, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z15, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z16, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z17, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z18, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z19, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x10_64_loop
	VZEROUPPER

mulGFNI_1x10_64_end:
	RET

// func mulAvxGFNI_1x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x10(SB), $0-88
	// Loading 4 of 10 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), DX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), R9
	MOVQ         96(BX), R10
	MOVQ         120(BX), R11
	MOVQ         144(BX), R12
	MOVQ         168(BX), R13
	MOVQ         192(BX), R14
	MOVQ         216(BX), BX
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, BX

	// Add start offset to input
	ADDQ R15, DX

mulAvxGFNI_1x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (DX), Y13
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y13, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y13, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y13, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y13, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y13, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y13, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y13, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y13, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y13, Y12
	VBROADCASTSD   72(CX), Y14
	VGF2P8AFFINEQB $0x00, Y14, Y13, Y13

	// Store 10 outputs
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y5, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x10_loop
	VZEROUPPER

mulAvxGFNI_1x10_end:
	RET

// func mulGFNI_1x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_1x10_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_1x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), CX
	MOVQ            out_base+48(FP), DX
	MOVQ            out_base+48(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, DX

	// Add start offset to input
	ADDQ R14, CX

mulGFNI_1x10_64Xor_loop:
	// Load 10 outputs
	VMOVDQU64 (BX), Z10
	VMOVDQU64 (SI), Z11
	VMOVDQU64 (DI), Z12
	VMOVDQU64 (R8), Z13
	VMOVDQU64 (R9), Z14
	VMOVDQU64 (R10), Z15
	VMOVDQU64 (R11), Z16
	VMOVDQU64 (R12), Z17
	VMOVDQU64 (R13), Z18
	VMOVDQU64 (DX), Z19

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (CX), Z20
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
	VXORPD         Z10, Z21, Z10
	VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
	VXORPD         Z11, Z21, Z11
	VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
	VXORPD         Z12, Z21, Z12
	VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
	VXORPD         Z13, Z21, Z13
	VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
	VXORPD         Z14, Z21, Z14
	VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
	VXORPD         Z15, Z21, Z15
	VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Store 10 outputs
	VMOVDQU64 Z10, (BX)
	ADDQ      $0x40, BX
	VMOVDQU64 Z11, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z12, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z13, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z14, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z15, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z16, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z17, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z18, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z19, (DX)
	ADDQ      $0x40, DX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_1x10_64Xor_loop
	VZEROUPPER

mulGFNI_1x10_64Xor_end:
	RET

// func mulAvxGFNI_1x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_1x10Xor(SB), $0-88
	// Loading 4 of 10 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_1x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), DX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), R9
	MOVQ         96(BX), R10
	MOVQ         120(BX), R11
	MOVQ         144(BX), R12
	MOVQ         168(BX), R13
	MOVQ         192(BX), R14
	MOVQ         216(BX), BX
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, BX

	// Add start offset to input
	ADDQ R15, DX

mulAvxGFNI_1x10Xor_loop:
	// Load 10 outputs
	VMOVDQU (SI), Y4
	VMOVDQU (DI), Y5
	VMOVDQU (R8), Y6
	VMOVDQU (R9), Y7
	VMOVDQU (R10), Y8
	VMOVDQU (R11), Y9
	VMOVDQU (R12), Y10
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (BX), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y5, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_1x10Xor_loop
	VZEROUPPER

mulAvxGFNI_1x10Xor_end:
	RET

// func mulAvxTwo_1x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_1x10Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_1x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), DX
	MOVQ  out_base+48(FP), BX
	MOVQ  (BX), SI
	MOVQ  24(BX), DI
	MOVQ  48(BX), R8
	MOVQ  72(BX), R9
	MOVQ  96(BX), R10
	MOVQ  120(BX), R11
	MOVQ  144(BX), R12
	MOVQ  168(BX), R13
	MOVQ  192(BX), R14
	MOVQ  216(BX), BX
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, BX

	// Add start offset to input
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_1x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (SI), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU (DI), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU (R8), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU (R9), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU (R10), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU (R11), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU (R12), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU (R13), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU (R14), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU (BX), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	VMOVDQU Y0, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y1, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y2, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y5, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y6, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y7, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y8, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y9, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_1x10Xor_loop
	VZEROUPPER

mulAvxTwo_1x10Xor_end:
	RET

// func mulAvxTwo_2x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ    n+80(FP), AX
	MOVQ    matrix_base+0(FP), CX
	SHRQ    $0x06, AX
	TESTQ   AX, AX
	JZ      mulAvxTwo_2x1_64_end
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1
	VMOVDQU 64(CX), Y2
	VMOVDQU 96(CX), Y3
	MOVQ    in_base+24(FP), CX
	MOVQ    (CX), DX
	MOVQ    24(CX), CX
	MOVQ    out_base+48(FP), BX
	MOVQ    (BX), BX
	MOVQ    start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX

	// Add start offset to input
	ADDQ         SI, DX
	ADDQ         SI, CX
	MOVQ         $0x0000000f, SI
	MOVQ         SI, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_2x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (DX), Y7
	VMOVDQU 32(DX), Y9
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y7, Y8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y7, Y7
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y8, Y8
	VPAND   Y6, Y10, Y10
	VPSHUFB Y7, Y0, Y7
	VPSHUFB Y9, Y0, Y9
	VPSHUFB Y8, Y1, Y8
	VPSHUFB Y10, Y1, Y10
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y9
	ADDQ    $0x40, CX
	VPSRLQ  $0x04, Y7, Y8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y7, Y7
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y8, Y8
	VPAND   Y6, Y10, Y10
	VPSHUFB Y7, Y2, Y7
	VPSHUFB Y9, Y2, Y9
	VPSHUFB Y8, Y3, Y8
	VPSHUFB Y10, Y3, Y10
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 1 outputs
	VMOVDQU Y4, (BX)
	VMOVDQU Y5, 32(BX)
	ADDQ    $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x1_64_loop
	VZEROUPPER

mulAvxTwo_2x1_64_end:
	RET

// func mulGFNI_2x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 5 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), BX
	MOVQ            start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX

	// Add start offset to input
	ADDQ SI, DX
	ADDQ SI, CX

mulGFNI_2x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z3
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z2

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (CX), Z3
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z1, Z3, Z3
	VXORPD         Z2, Z3, Z2

	// Store 1 outputs
	VMOVDQU64 Z2, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x1_64_loop
	VZEROUPPER

mulGFNI_2x1_64_end:
	RET

// func mulAvxGFNI_2x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 5 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), BX
	MOVQ         start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX

	// Add start offset to input
	ADDQ SI, DX
	ADDQ SI, CX

mulAvxGFNI_2x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y3
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y3, Y2

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (CX), Y3
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y1, Y3, Y3
	VXORPD         Y2, Y3, Y2

	// Store 1 outputs
	VMOVDQU Y2, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x1_loop
	VZEROUPPER

mulAvxGFNI_2x1_end:
	RET

// func mulGFNI_2x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 5 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), BX
	MOVQ            start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX

	// Add start offset to input
	ADDQ SI, DX
	ADDQ SI, CX

mulGFNI_2x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (BX), Z2

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z3
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z3
	VXORPD         Z2, Z3, Z2

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (CX), Z3
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z1, Z3, Z3
	VXORPD         Z2, Z3, Z2

	// Store 1 outputs
	VMOVDQU64 Z2, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x1_64Xor_loop
	VZEROUPPER

mulGFNI_2x1_64Xor_end:
	RET

// func mulAvxGFNI_2x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 5 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), BX
	MOVQ         start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX

	// Add start offset to input
	ADDQ SI, DX
	ADDQ SI, CX

mulAvxGFNI_2x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (BX), Y2

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y3
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y3, Y3
	VXORPD         Y2, Y3, Y2

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (CX), Y3
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y1, Y3, Y3
	VXORPD         Y2, Y3, Y2

	// Store 1 outputs
	VMOVDQU Y2, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x1Xor_loop
	VZEROUPPER

mulAvxGFNI_2x1Xor_end:
	RET

// func mulAvxTwo_2x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ    n+80(FP), AX
	MOVQ    matrix_base+0(FP), CX
	SHRQ    $0x06, AX
	TESTQ   AX, AX
	JZ      mulAvxTwo_2x1_64Xor_end
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1
	VMOVDQU 64(CX), Y2
	VMOVDQU 96(CX), Y3
	MOVQ    in_base+24(FP), CX
	MOVQ    (CX), DX
	MOVQ    24(CX), CX
	MOVQ    out_base+48(FP), BX
	MOVQ    (BX), BX
	MOVQ    start+72(FP), SI

	// Add start offset to output
	ADDQ SI, BX

	// Add start offset to input
	ADDQ         SI, DX
	ADDQ         SI, CX
	MOVQ         $0x0000000f, SI
	MOVQ         SI, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_2x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (BX), Y4
	VMOVDQU 32(BX), Y5

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (DX), Y7
	VMOVDQU 32(DX), Y9
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y7, Y8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y7, Y7
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y8, Y8
	VPAND   Y6, Y10, Y10
	VPSHUFB Y7, Y0, Y7
	VPSHUFB Y9, Y0, Y9
	VPSHUFB Y8, Y1, Y8
	VPSHUFB Y10, Y1, Y10
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y9
	ADDQ    $0x40, CX
	VPSRLQ  $0x04, Y7, Y8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y7, Y7
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y8, Y8
	VPAND   Y6, Y10, Y10
	VPSHUFB Y7, Y2, Y7
	VPSHUFB Y9, Y2, Y9
	VPSHUFB Y8, Y3, Y8
	VPSHUFB Y10, Y3, Y10
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 1 outputs
	VMOVDQU Y4, (BX)
	VMOVDQU Y5, 32(BX)
	ADDQ    $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_2x1_64Xor_end:
	RET

// func mulAvxTwo_2x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 25 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), SI
	MOVQ  start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI
	ADDQ R8, SI

	// Add start offset to input
	ADDQ         R8, BX
	ADDQ         R8, DX
	MOVQ         $0x0000000f, R8
	MOVQ         R8, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_2x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (DI)
	VMOVDQU Y1, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y2, (SI)
	VMOVDQU Y3, 32(SI)
	ADDQ    $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x2_64_loop
	VZEROUPPER

mulAvxTwo_2x2_64_end:
	RET

// func mulGFNI_2x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), BX
	MOVQ            start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI
	ADDQ DI, BX

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, CX

mulGFNI_2x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z6
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z6, Z4
	VGF2P8AFFINEQB $0x00, Z1, Z6, Z5

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (CX), Z6
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
	VXORPD         Z4, Z7, Z4
	VGF2P8AFFINEQB $0x00, Z3, Z6, Z7
	VXORPD         Z5, Z7, Z5

	// Store 2 outputs
	VMOVDQU64 Z4, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z5, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x2_64_loop
	VZEROUPPER

mulGFNI_2x2_64_end:
	RET

// func mulAvxGFNI_2x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x2(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), BX
	MOVQ         start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI
	ADDQ DI, BX

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, CX

mulAvxGFNI_2x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y6
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y6, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y6, Y5

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (CX), Y6
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y2, Y6, Y7
	VXORPD         Y4, Y7, Y4
	VGF2P8AFFINEQB $0x00, Y3, Y6, Y7
	VXORPD         Y5, Y7, Y5

	// Store 2 outputs
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y5, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x2_loop
	VZEROUPPER

mulAvxGFNI_2x2_end:
	RET

// func mulGFNI_2x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), BX
	MOVQ            start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI
	ADDQ DI, BX

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, CX

mulGFNI_2x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (SI), Z4
	VMOVDQU64 (BX), Z5

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z6
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z6, Z7
	VXORPD         Z4, Z7, Z4
	VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
	VXORPD         Z5, Z7, Z5

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (CX), Z6
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
	VXORPD         Z4, Z7, Z4
	VGF2P8AFFINEQB $0x00, Z3, Z6, Z7
	VXORPD         Z5, Z7, Z5

	// Store 2 outputs
	VMOVDQU64 Z4, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z5, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x2_64Xor_loop
	VZEROUPPER

mulGFNI_2x2_64Xor_end:
	RET

// func mulAvxGFNI_2x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x2Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), BX
	MOVQ         start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI
	ADDQ DI, BX

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, CX

mulAvxGFNI_2x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (SI), Y4
	VMOVDQU (BX), Y5

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y6
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y6, Y7
	VXORPD         Y4, Y7, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y6, Y7
	VXORPD         Y5, Y7, Y5

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (CX), Y6
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y2, Y6, Y7
	VXORPD         Y4, Y7, Y4
	VGF2P8AFFINEQB $0x00, Y3, Y6, Y7
	VXORPD         Y5, Y7, Y5

	// Store 2 outputs
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y5, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x2Xor_loop
	VZEROUPPER

mulAvxGFNI_2x2Xor_end:
	RET

// func mulAvxTwo_2x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 25 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), SI
	MOVQ  start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI
	ADDQ R8, SI

	// Add start offset to input
	ADDQ         R8, BX
	ADDQ         R8, DX
	MOVQ         $0x0000000f, R8
	MOVQ         R8, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_2x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (DI), Y0
	VMOVDQU 32(DI), Y1
	VMOVDQU (SI), Y2
	VMOVDQU 32(SI), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (DI)
	VMOVDQU Y1, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y2, (SI)
	VMOVDQU Y3, 32(SI)
	ADDQ    $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_2x2_64Xor_end:
	RET

// func mulAvxTwo_2x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), SI
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, SI

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_2x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (DI)
	VMOVDQU Y1, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y2, (R8)
	VMOVDQU Y3, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y4, (SI)
	VMOVDQU Y5, 32(SI)
	ADDQ    $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x3_64_loop
	VZEROUPPER

mulAvxTwo_2x3_64_end:
	RET

// func mulGFNI_2x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), BX
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, BX

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, CX

mulGFNI_2x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z9
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z1, Z9, Z7
	VGF2P8AFFINEQB $0x00, Z2, Z9, Z8

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (CX), Z9
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z3, Z9, Z10
	VXORPD         Z6, Z10, Z6
	VGF2P8AFFINEQB $0x00, Z4, Z9, Z10
	VXORPD         Z7, Z10, Z7
	VGF2P8AFFINEQB $0x00, Z5, Z9, Z10
	VXORPD         Z8, Z10, Z8

	// Store 3 outputs
	VMOVDQU64 Z6, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z7, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z8, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x3_64_loop
	VZEROUPPER

mulGFNI_2x3_64_end:
	RET

// func mulAvxGFNI_2x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x3(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), BX
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, BX

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, CX

mulAvxGFNI_2x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (DX), Y9
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y9, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y9, Y8

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (CX), Y9
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y3, Y9, Y10
	VXORPD         Y6, Y10, Y6
	VGF2P8AFFINEQB $0x00, Y4, Y9, Y10
	VXORPD         Y7, Y10, Y7
	VGF2P8AFFINEQB $0x00, Y5, Y9, Y10
	VXORPD         Y8, Y10, Y8

	// Store 3 outputs
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x3_loop
	VZEROUPPER

mulAvxGFNI_2x3_end:
	RET

// func mulGFNI_2x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), BX
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, BX

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, CX

mulGFNI_2x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (SI), Z6
	VMOVDQU64 (DI), Z7
	VMOVDQU64 (BX), Z8

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z9
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z9, Z10
	VXORPD         Z6, Z10, Z6
	VGF2P8AFFINEQB $0x00, Z1, Z9, Z10
	VXORPD         Z7, Z10, Z7
	VGF2P8AFFINEQB $0x00, Z2, Z9, Z10
	VXORPD         Z8, Z10, Z8

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (CX), Z9
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z3, Z9, Z10
	VXORPD         Z6, Z10, Z6
	VGF2P8AFFINEQB $0x00, Z4, Z9, Z10
	VXORPD         Z7, Z10, Z7
	VGF2P8AFFINEQB $0x00, Z5, Z9, Z10
	VXORPD         Z8, Z10, Z8

	// Store 3 outputs
	VMOVDQU64 Z6, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z7, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z8, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x3_64Xor_loop
	VZEROUPPER

mulGFNI_2x3_64Xor_end:
	RET

// func mulAvxGFNI_2x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x3Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), BX
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, SI
	ADDQ R8, DI
	ADDQ R8, BX

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, CX

mulAvxGFNI_2x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (SI), Y6
	VMOVDQU (DI), Y7
	VMOVDQU (BX), Y8

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (DX), Y9
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y9, Y10
	VXORPD         Y6, Y10, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y9, Y10
	VXORPD         Y7, Y10, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y9, Y10
	VXORPD         Y8, Y10, Y8

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (CX), Y9
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y3, Y9, Y10
	VXORPD         Y6, Y10, Y6
	VGF2P8AFFINEQB $0x00, Y4, Y9, Y10
	VXORPD         Y7, Y10, Y7
	VGF2P8AFFINEQB $0x00, Y5, Y9, Y10
	VXORPD         Y8, Y10, Y8

	// Store 3 outputs
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x3Xor_loop
	VZEROUPPER

mulAvxGFNI_2x3Xor_end:
	RET

// func mulAvxTwo_2x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), SI
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, SI

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_2x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (DI), Y0
	VMOVDQU 32(DI), Y1
	VMOVDQU (R8), Y2
	VMOVDQU 32(R8), Y3
	VMOVDQU (SI), Y4
	VMOVDQU 32(SI), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (DI)
	VMOVDQU Y1, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y2, (R8)
	VMOVDQU Y3, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y4, (SI)
	VMOVDQU Y5, 32(SI)
	ADDQ    $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_2x3_64Xor_end:
	RET

// func mulAvxTwo_2x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x4(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 25 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), SI
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, SI

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_2x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x4_loop
	VZEROUPPER

mulAvxTwo_2x4_end:
	RET

// func mulGFNI_2x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x4_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), BX
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, BX

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, CX

mulGFNI_2x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z12
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z12, Z8
	VGF2P8AFFINEQB $0x00, Z1, Z12, Z9
	VGF2P8AFFINEQB $0x00, Z2, Z12, Z10
	VGF2P8AFFINEQB $0x00, Z3, Z12, Z11

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (CX), Z12
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
	VXORPD         Z8, Z13, Z8
	VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Store 4 outputs
	VMOVDQU64 Z8, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z10, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z11, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x4_64_loop
	VZEROUPPER

mulGFNI_2x4_64_end:
	RET

// func mulAvxGFNI_2x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x4(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), BX
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, BX

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, CX

mulAvxGFNI_2x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (DX), Y12
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y12, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y12, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y12, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y12, Y11

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (CX), Y12
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
	VXORPD         Y8, Y13, Y8
	VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Store 4 outputs
	VMOVDQU Y8, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x4_loop
	VZEROUPPER

mulAvxGFNI_2x4_end:
	RET

// func mulGFNI_2x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x4_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), BX
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, BX

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, CX

mulGFNI_2x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (SI), Z8
	VMOVDQU64 (DI), Z9
	VMOVDQU64 (R8), Z10
	VMOVDQU64 (BX), Z11

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z12
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
	VXORPD         Z8, Z13, Z8
	VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (CX), Z12
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
	VXORPD         Z8, Z13, Z8
	VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Store 4 outputs
	VMOVDQU64 Z8, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z10, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z11, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x4_64Xor_loop
	VZEROUPPER

mulGFNI_2x4_64Xor_end:
	RET

// func mulAvxGFNI_2x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x4Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), CX
	MOVQ         out_base+48(FP), BX
	MOVQ         out_base+48(FP), BX
	MOVQ         (BX), SI
	MOVQ         24(BX), DI
	MOVQ         48(BX), R8
	MOVQ         72(BX), BX
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, BX

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, CX

mulAvxGFNI_2x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (SI), Y8
	VMOVDQU (DI), Y9
	VMOVDQU (R8), Y10
	VMOVDQU (BX), Y11

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (DX), Y12
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
	VXORPD         Y8, Y13, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (CX), Y12
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
	VXORPD         Y8, Y13, Y8
	VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Store 4 outputs
	VMOVDQU Y8, (SI)
	ADDQ    $0x20, SI
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (BX)
	ADDQ    $0x20, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x4Xor_loop
	VZEROUPPER

mulAvxGFNI_2x4Xor_end:
	RET

// func mulAvxTwo_2x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x4Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 25 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), SI
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, SI

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_2x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (SI), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x4Xor_loop
	VZEROUPPER

mulAvxTwo_2x4Xor_end:
	RET

// func mulAvxTwo_2x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x5(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), SI
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, SI

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_2x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x5_loop
	VZEROUPPER

mulAvxTwo_2x5_end:
	RET

// func mulGFNI_2x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x5_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), BX
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, BX

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, CX

mulGFNI_2x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z15
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z15, Z10
	VGF2P8AFFINEQB $0x00, Z1, Z15, Z11
	VGF2P8AFFINEQB $0x00, Z2, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z3, Z15, Z13
	VGF2P8AFFINEQB $0x00, Z4, Z15, Z14

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (CX), Z15
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
	VXORPD         Z10, Z16, Z10
	VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
	VXORPD         Z11, Z16, Z11
	VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Store 5 outputs
	VMOVDQU64 Z10, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z11, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z12, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z14, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x5_64_loop
	VZEROUPPER

mulGFNI_2x5_64_end:
	RET

// func mulAvxGFNI_2x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x5(SB), $0-88
	// Loading 9 of 10 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), SI
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, SI

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, DX

mulAvxGFNI_2x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x5_loop
	VZEROUPPER

mulAvxGFNI_2x5_end:
	RET

// func mulGFNI_2x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x5_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), BX
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, BX

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, CX

mulGFNI_2x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (SI), Z10
	VMOVDQU64 (DI), Z11
	VMOVDQU64 (R8), Z12
	VMOVDQU64 (R9), Z13
	VMOVDQU64 (BX), Z14

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z15
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z15, Z16
	VXORPD         Z10, Z16, Z10
	VGF2P8AFFINEQB $0x00, Z1, Z15, Z16
	VXORPD         Z11, Z16, Z11
	VGF2P8AFFINEQB $0x00, Z2, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z3, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z4, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (CX), Z15
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
	VXORPD         Z10, Z16, Z10
	VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
	VXORPD         Z11, Z16, Z11
	VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Store 5 outputs
	VMOVDQU64 Z10, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z11, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z12, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z14, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x5_64Xor_loop
	VZEROUPPER

mulGFNI_2x5_64Xor_end:
	RET

// func mulAvxGFNI_2x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x5Xor(SB), $0-88
	// Loading 9 of 10 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), SI
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, SI

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, DX

mulAvxGFNI_2x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (DI), Y9
	VMOVDQU (R8), Y10
	VMOVDQU (R9), Y11
	VMOVDQU (R10), Y12
	VMOVDQU (SI), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x5Xor_loop
	VZEROUPPER

mulAvxGFNI_2x5Xor_end:
	RET

// func mulAvxTwo_2x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x5Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), SI
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, SI

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_2x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (SI), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x5Xor_loop
	VZEROUPPER

mulAvxTwo_2x5Xor_end:
	RET

// func mulAvxTwo_2x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x6(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), SI
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, SI

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_2x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x6_loop
	VZEROUPPER

mulAvxTwo_2x6_end:
	RET

// func mulGFNI_2x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x6_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), BX
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, BX

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, CX

mulGFNI_2x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z18
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z18, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z18, Z13
	VGF2P8AFFINEQB $0x00, Z2, Z18, Z14
	VGF2P8AFFINEQB $0x00, Z3, Z18, Z15
	VGF2P8AFFINEQB $0x00, Z4, Z18, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z18, Z17

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (CX), Z18
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
	VXORPD         Z12, Z19, Z12
	VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
	VXORPD         Z13, Z19, Z13
	VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
	VXORPD         Z14, Z19, Z14
	VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Store 6 outputs
	VMOVDQU64 Z12, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z13, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z14, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z15, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z16, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z17, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x6_64_loop
	VZEROUPPER

mulGFNI_2x6_64_end:
	RET

// func mulAvxGFNI_2x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x6(SB), $0-88
	// Loading 8 of 12 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), SI
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, SI

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, DX

mulAvxGFNI_2x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y9, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x6_loop
	VZEROUPPER

mulAvxGFNI_2x6_end:
	RET

// func mulGFNI_2x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x6_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), BX
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, BX

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, CX

mulGFNI_2x6_64Xor_loop:
	// Load 6 outputs
	VMOVDQU64 (SI), Z12
	VMOVDQU64 (DI), Z13
	VMOVDQU64 (R8), Z14
	VMOVDQU64 (R9), Z15
	VMOVDQU64 (R10), Z16
	VMOVDQU64 (BX), Z17

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z18
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
	VXORPD         Z12, Z19, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
	VXORPD         Z13, Z19, Z13
	VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
	VXORPD         Z14, Z19, Z14
	VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (CX), Z18
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
	VXORPD         Z12, Z19, Z12
	VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
	VXORPD         Z13, Z19, Z13
	VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
	VXORPD         Z14, Z19, Z14
	VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Store 6 outputs
	VMOVDQU64 Z12, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z13, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z14, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z15, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z16, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z17, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x6_64Xor_loop
	VZEROUPPER

mulGFNI_2x6_64Xor_end:
	RET

// func mulAvxGFNI_2x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x6Xor(SB), $0-88
	// Loading 8 of 12 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), SI
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, SI

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, DX

mulAvxGFNI_2x6Xor_loop:
	// Load 6 outputs
	VMOVDQU (DI), Y8
	VMOVDQU (R8), Y9
	VMOVDQU (R9), Y10
	VMOVDQU (R10), Y11
	VMOVDQU (R11), Y12
	VMOVDQU (SI), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y9, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x6Xor_loop
	VZEROUPPER

mulAvxGFNI_2x6Xor_end:
	RET

// func mulAvxTwo_2x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x6Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), SI
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, SI

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_2x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU (SI), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x6Xor_loop
	VZEROUPPER

mulAvxTwo_2x6Xor_end:
	RET

// func mulAvxTwo_2x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x7(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 40 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), SI
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, SI

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_2x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x7_loop
	VZEROUPPER

mulAvxTwo_2x7_end:
	RET

// func mulGFNI_2x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x7_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), BX
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, BX

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, CX

mulGFNI_2x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (DX), Z21
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z21, Z14
	VGF2P8AFFINEQB $0x00, Z1, Z21, Z15
	VGF2P8AFFINEQB $0x00, Z2, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z3, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z4, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z5, Z21, Z19
	VGF2P8AFFINEQB $0x00, Z6, Z21, Z20

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (CX), Z21
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
	VXORPD         Z14, Z22, Z14
	VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
	VXORPD         Z15, Z22, Z15
	VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
	VXORPD         Z16, Z22, Z16
	VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
	VXORPD         Z17, Z22, Z17
	VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Store 7 outputs
	VMOVDQU64 Z14, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z15, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z16, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z17, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z20, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x7_64_loop
	VZEROUPPER

mulGFNI_2x7_64_end:
	RET

// func mulAvxGFNI_2x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x7(SB), $0-88
	// Loading 7 of 14 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), SI
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, SI

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, DX

mulAvxGFNI_2x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x7_loop
	VZEROUPPER

mulAvxGFNI_2x7_end:
	RET

// func mulGFNI_2x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x7_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), BX
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, BX

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, CX

mulGFNI_2x7_64Xor_loop:
	// Load 7 outputs
	VMOVDQU64 (SI), Z14
	VMOVDQU64 (DI), Z15
	VMOVDQU64 (R8), Z16
	VMOVDQU64 (R9), Z17
	VMOVDQU64 (R10), Z18
	VMOVDQU64 (R11), Z19
	VMOVDQU64 (BX), Z20

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (DX), Z21
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z21, Z22
	VXORPD         Z14, Z22, Z14
	VGF2P8AFFINEQB $0x00, Z1, Z21, Z22
	VXORPD         Z15, Z22, Z15
	VGF2P8AFFINEQB $0x00, Z2, Z21, Z22
	VXORPD         Z16, Z22, Z16
	VGF2P8AFFINEQB $0x00, Z3, Z21, Z22
	VXORPD         Z17, Z22, Z17
	VGF2P8AFFINEQB $0x00, Z4, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z5, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z6, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (CX), Z21
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
	VXORPD         Z14, Z22, Z14
	VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
	VXORPD         Z15, Z22, Z15
	VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
	VXORPD         Z16, Z22, Z16
	VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
	VXORPD         Z17, Z22, Z17
	VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Store 7 outputs
	VMOVDQU64 Z14, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z15, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z16, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z17, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z20, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x7_64Xor_loop
	VZEROUPPER

mulGFNI_2x7_64Xor_end:
	RET

// func mulAvxGFNI_2x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x7Xor(SB), $0-88
	// Loading 7 of 14 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), SI
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, SI

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, DX

mulAvxGFNI_2x7Xor_loop:
	// Load 7 outputs
	VMOVDQU (DI), Y7
	VMOVDQU (R8), Y8
	VMOVDQU (R9), Y9
	VMOVDQU (R10), Y10
	VMOVDQU (R11), Y11
	VMOVDQU (R12), Y12
	VMOVDQU (SI), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x7Xor_loop
	VZEROUPPER

mulAvxGFNI_2x7Xor_end:
	RET

// func mulAvxTwo_2x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x7Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 40 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), SI
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, SI

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_2x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU (R12), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU (SI), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x7Xor_loop
	VZEROUPPER

mulAvxTwo_2x7Xor_end:
	RET

// func mulAvxTwo_2x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x8(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 45 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), SI
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, SI

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_2x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x8_loop
	VZEROUPPER

mulAvxTwo_2x8_end:
	RET

// func mulGFNI_2x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x8_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), R12
	MOVQ            168(BX), BX
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, BX

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, CX

mulGFNI_2x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z16
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z17
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z18
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z19
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z20
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z21
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z22
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z23

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z16, Z25, Z16
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z17, Z25, Z17
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 8 outputs
	VMOVDQU64 Z16, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z17, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z18, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z19, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z20, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z21, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z22, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z23, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x8_64_loop
	VZEROUPPER

mulGFNI_2x8_64_end:
	RET

// func mulAvxGFNI_2x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x8(SB), $0-88
	// Loading 6 of 16 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), SI
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, SI

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, DX

mulAvxGFNI_2x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x8_loop
	VZEROUPPER

mulAvxGFNI_2x8_end:
	RET

// func mulGFNI_2x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x8_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), R12
	MOVQ            168(BX), BX
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, BX

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, CX

mulGFNI_2x8_64Xor_loop:
	// Load 8 outputs
	VMOVDQU64 (SI), Z16
	VMOVDQU64 (DI), Z17
	VMOVDQU64 (R8), Z18
	VMOVDQU64 (R9), Z19
	VMOVDQU64 (R10), Z20
	VMOVDQU64 (R11), Z21
	VMOVDQU64 (R12), Z22
	VMOVDQU64 (BX), Z23

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
	VXORPD         Z16, Z25, Z16
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
	VXORPD         Z17, Z25, Z17
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z16, Z25, Z16
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z17, Z25, Z17
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 8 outputs
	VMOVDQU64 Z16, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z17, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z18, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z19, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z20, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z21, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z22, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z23, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x8_64Xor_loop
	VZEROUPPER

mulGFNI_2x8_64Xor_end:
	RET

// func mulAvxGFNI_2x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x8Xor(SB), $0-88
	// Loading 6 of 16 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), SI
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, SI

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, DX

mulAvxGFNI_2x8Xor_loop:
	// Load 8 outputs
	VMOVDQU (DI), Y6
	VMOVDQU (R8), Y7
	VMOVDQU (R9), Y8
	VMOVDQU (R10), Y9
	VMOVDQU (R11), Y10
	VMOVDQU (R12), Y11
	VMOVDQU (R13), Y12
	VMOVDQU (SI), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x8Xor_loop
	VZEROUPPER

mulAvxGFNI_2x8Xor_end:
	RET

// func mulAvxTwo_2x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x8Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 45 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), SI
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, SI

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_2x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU (R12), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU (R13), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU (SI), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x8Xor_loop
	VZEROUPPER

mulAvxTwo_2x8Xor_end:
	RET

// func mulAvxTwo_2x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x9(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), R14
	MOVQ  192(SI), SI
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, SI

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_2x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y8, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x9_loop
	VZEROUPPER

mulAvxTwo_2x9_end:
	RET

// func mulGFNI_2x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x9_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), R12
	MOVQ            168(BX), R13
	MOVQ            192(BX), BX
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, BX

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, CX

mulGFNI_2x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (DX), Z27
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z27, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z27, Z19
	VGF2P8AFFINEQB $0x00, Z2, Z27, Z20
	VGF2P8AFFINEQB $0x00, Z3, Z27, Z21
	VGF2P8AFFINEQB $0x00, Z4, Z27, Z22
	VGF2P8AFFINEQB $0x00, Z5, Z27, Z23
	VGF2P8AFFINEQB $0x00, Z6, Z27, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z27, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z27, Z26

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (CX), Z27
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
	VXORPD         Z18, Z28, Z18
	VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
	VXORPD         Z19, Z28, Z19
	VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
	VXORPD         Z20, Z28, Z20
	VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
	VXORPD         Z21, Z28, Z21
	VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
	VXORPD         Z22, Z28, Z22
	VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
	VXORPD         Z23, Z28, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Store 9 outputs
	VMOVDQU64 Z18, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z19, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z20, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z21, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z22, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z23, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z24, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z25, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z26, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x9_64_loop
	VZEROUPPER

mulGFNI_2x9_64_end:
	RET

// func mulAvxGFNI_2x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x9(SB), $0-88
	// Loading 5 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), R14
	MOVQ         192(SI), SI
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, SI

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, DX

mulAvxGFNI_2x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	VMOVDQU Y5, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x9_loop
	VZEROUPPER

mulAvxGFNI_2x9_end:
	RET

// func mulGFNI_2x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x9_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), R12
	MOVQ            168(BX), R13
	MOVQ            192(BX), BX
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, BX

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, CX

mulGFNI_2x9_64Xor_loop:
	// Load 9 outputs
	VMOVDQU64 (SI), Z18
	VMOVDQU64 (DI), Z19
	VMOVDQU64 (R8), Z20
	VMOVDQU64 (R9), Z21
	VMOVDQU64 (R10), Z22
	VMOVDQU64 (R11), Z23
	VMOVDQU64 (R12), Z24
	VMOVDQU64 (R13), Z25
	VMOVDQU64 (BX), Z26

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (DX), Z27
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z27, Z28
	VXORPD         Z18, Z28, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z27, Z28
	VXORPD         Z19, Z28, Z19
	VGF2P8AFFINEQB $0x00, Z2, Z27, Z28
	VXORPD         Z20, Z28, Z20
	VGF2P8AFFINEQB $0x00, Z3, Z27, Z28
	VXORPD         Z21, Z28, Z21
	VGF2P8AFFINEQB $0x00, Z4, Z27, Z28
	VXORPD         Z22, Z28, Z22
	VGF2P8AFFINEQB $0x00, Z5, Z27, Z28
	VXORPD         Z23, Z28, Z23
	VGF2P8AFFINEQB $0x00, Z6, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (CX), Z27
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
	VXORPD         Z18, Z28, Z18
	VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
	VXORPD         Z19, Z28, Z19
	VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
	VXORPD         Z20, Z28, Z20
	VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
	VXORPD         Z21, Z28, Z21
	VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
	VXORPD         Z22, Z28, Z22
	VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
	VXORPD         Z23, Z28, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Store 9 outputs
	VMOVDQU64 Z18, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z19, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z20, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z21, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z22, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z23, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z24, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z25, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z26, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x9_64Xor_loop
	VZEROUPPER

mulGFNI_2x9_64Xor_end:
	RET

// func mulAvxGFNI_2x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x9Xor(SB), $0-88
	// Loading 5 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), R14
	MOVQ         192(SI), SI
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, SI

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, DX

mulAvxGFNI_2x9Xor_loop:
	// Load 9 outputs
	VMOVDQU (DI), Y5
	VMOVDQU (R8), Y6
	VMOVDQU (R9), Y7
	VMOVDQU (R10), Y8
	VMOVDQU (R11), Y9
	VMOVDQU (R12), Y10
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (SI), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	VMOVDQU Y5, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x9Xor_loop
	VZEROUPPER

mulAvxGFNI_2x9Xor_end:
	RET

// func mulAvxTwo_2x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x9Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), R14
	MOVQ  192(SI), SI
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, SI

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_2x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU (R12), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU (R13), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU (R14), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU (SI), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y8, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x9Xor_loop
	VZEROUPPER

mulAvxTwo_2x9Xor_end:
	RET

// func mulAvxTwo_2x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x10(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 55 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), R14
	MOVQ  192(SI), R15
	MOVQ  216(SI), SI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_2x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y8, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y9, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x10_loop
	VZEROUPPER

mulAvxTwo_2x10_end:
	RET

// func mulGFNI_2x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x10_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), R12
	MOVQ            168(BX), R13
	MOVQ            192(BX), R14
	MOVQ            216(BX), BX
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, BX

	// Add start offset to input
	ADDQ R15, DX
	ADDQ R15, CX

mulGFNI_2x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 10 outputs
	VMOVDQU64 Z20, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z21, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z22, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x10_64_loop
	VZEROUPPER

mulGFNI_2x10_64_end:
	RET

// func mulAvxGFNI_2x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x10(SB), $8-88
	// Loading 4 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), R14
	MOVQ         192(SI), R15
	MOVQ         216(SI), SI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, DX

mulAvxGFNI_2x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x10_loop
	VZEROUPPER

mulAvxGFNI_2x10_end:
	RET

// func mulGFNI_2x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_2x10_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_2x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), CX
	MOVQ            out_base+48(FP), BX
	MOVQ            out_base+48(FP), BX
	MOVQ            (BX), SI
	MOVQ            24(BX), DI
	MOVQ            48(BX), R8
	MOVQ            72(BX), R9
	MOVQ            96(BX), R10
	MOVQ            120(BX), R11
	MOVQ            144(BX), R12
	MOVQ            168(BX), R13
	MOVQ            192(BX), R14
	MOVQ            216(BX), BX
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, BX

	// Add start offset to input
	ADDQ R15, DX
	ADDQ R15, CX

mulGFNI_2x10_64Xor_loop:
	// Load 10 outputs
	VMOVDQU64 (SI), Z20
	VMOVDQU64 (DI), Z21
	VMOVDQU64 (R8), Z22
	VMOVDQU64 (R9), Z23
	VMOVDQU64 (R10), Z24
	VMOVDQU64 (R11), Z25
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (R13), Z27
	VMOVDQU64 (R14), Z28
	VMOVDQU64 (BX), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 10 outputs
	VMOVDQU64 Z20, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z21, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z22, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (BX)
	ADDQ      $0x40, BX

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_2x10_64Xor_loop
	VZEROUPPER

mulGFNI_2x10_64Xor_end:
	RET

// func mulAvxGFNI_2x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_2x10Xor(SB), $8-88
	// Loading 4 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_2x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), DX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), R14
	MOVQ         192(SI), R15
	MOVQ         216(SI), SI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, DX

mulAvxGFNI_2x10Xor_loop:
	// Load 10 outputs
	VMOVDQU (DI), Y4
	VMOVDQU (R8), Y5
	VMOVDQU (R9), Y6
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (SI), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_2x10Xor_loop
	VZEROUPPER

mulAvxGFNI_2x10Xor_end:
	RET

// func mulAvxTwo_2x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_2x10Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 55 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_2x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), DX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), R14
	MOVQ  192(SI), R15
	MOVQ  216(SI), SI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_2x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU (R12), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU (R13), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU (R14), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU (R15), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU (SI), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y8, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y9, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_2x10Xor_loop
	VZEROUPPER

mulAvxTwo_2x10Xor_end:
	RET

// func mulAvxTwo_3x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), DI
	MOVQ  start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI

	// Add start offset to input
	ADDQ         R8, BX
	ADDQ         R8, SI
	ADDQ         R8, DX
	MOVQ         $0x0000000f, R8
	MOVQ         R8, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_3x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (DI)
	VMOVDQU Y1, 32(DI)
	ADDQ    $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x1_64_loop
	VZEROUPPER

mulAvxTwo_3x1_64_end:
	RET

// func mulGFNI_3x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), SI
	MOVQ            start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, BX
	ADDQ DI, CX

mulGFNI_3x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z4
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z3

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z4
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z4, Z4
	VXORPD         Z3, Z4, Z3

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (CX), Z4
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z2, Z4, Z4
	VXORPD         Z3, Z4, Z3

	// Store 1 outputs
	VMOVDQU64 Z3, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x1_64_loop
	VZEROUPPER

mulGFNI_3x1_64_end:
	RET

// func mulAvxGFNI_3x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), CX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), SI
	MOVQ         start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, BX
	ADDQ DI, CX

mulAvxGFNI_3x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y4
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y4, Y3

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y4
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y4, Y4
	VXORPD         Y3, Y4, Y3

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (CX), Y4
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y2, Y4, Y4
	VXORPD         Y3, Y4, Y3

	// Store 1 outputs
	VMOVDQU Y3, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x1_loop
	VZEROUPPER

mulAvxGFNI_3x1_end:
	RET

// func mulGFNI_3x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), SI
	MOVQ            start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, BX
	ADDQ DI, CX

mulGFNI_3x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (SI), Z3

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z4
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z4
	VXORPD         Z3, Z4, Z3

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z4
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z4, Z4
	VXORPD         Z3, Z4, Z3

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (CX), Z4
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z2, Z4, Z4
	VXORPD         Z3, Z4, Z3

	// Store 1 outputs
	VMOVDQU64 Z3, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x1_64Xor_loop
	VZEROUPPER

mulGFNI_3x1_64Xor_end:
	RET

// func mulAvxGFNI_3x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 6 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), CX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), SI
	MOVQ         start+72(FP), DI

	// Add start offset to output
	ADDQ DI, SI

	// Add start offset to input
	ADDQ DI, DX
	ADDQ DI, BX
	ADDQ DI, CX

mulAvxGFNI_3x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (SI), Y3

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y4
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y4, Y4
	VXORPD         Y3, Y4, Y3

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y4
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y4, Y4
	VXORPD         Y3, Y4, Y3

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (CX), Y4
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y2, Y4, Y4
	VXORPD         Y3, Y4, Y3

	// Store 1 outputs
	VMOVDQU Y3, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x1Xor_loop
	VZEROUPPER

mulAvxGFNI_3x1Xor_end:
	RET

// func mulAvxTwo_3x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), DI
	MOVQ  start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI

	// Add start offset to input
	ADDQ         R8, BX
	ADDQ         R8, SI
	ADDQ         R8, DX
	MOVQ         $0x0000000f, R8
	MOVQ         R8, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_3x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (DI), Y0
	VMOVDQU 32(DI), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (DI)
	VMOVDQU Y1, 32(DI)
	ADDQ    $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_3x1_64Xor_end:
	RET

// func mulAvxTwo_3x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 33 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), DI
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8
	ADDQ R9, DI

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, SI
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_3x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R8)
	VMOVDQU Y1, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x2_64_loop
	VZEROUPPER

mulAvxTwo_3x2_64_end:
	RET

// func mulGFNI_3x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), SI
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI
	ADDQ R8, SI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, CX

mulGFNI_3x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z8
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z8, Z6
	VGF2P8AFFINEQB $0x00, Z1, Z8, Z7

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z8
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z8, Z9
	VXORPD         Z6, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z3, Z8, Z9
	VXORPD         Z7, Z9, Z7

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (CX), Z8
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z4, Z8, Z9
	VXORPD         Z6, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z5, Z8, Z9
	VXORPD         Z7, Z9, Z7

	// Store 2 outputs
	VMOVDQU64 Z6, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z7, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x2_64_loop
	VZEROUPPER

mulGFNI_3x2_64_end:
	RET

// func mulAvxGFNI_3x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x2(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), CX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), SI
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI
	ADDQ R8, SI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, CX

mulAvxGFNI_3x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y8
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y8, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y8, Y7

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y8
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y8, Y9
	VXORPD         Y6, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y8, Y9
	VXORPD         Y7, Y9, Y7

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (CX), Y8
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y4, Y8, Y9
	VXORPD         Y6, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y5, Y8, Y9
	VXORPD         Y7, Y9, Y7

	// Store 2 outputs
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x2_loop
	VZEROUPPER

mulAvxGFNI_3x2_end:
	RET

// func mulGFNI_3x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), SI
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI
	ADDQ R8, SI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, CX

mulGFNI_3x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (DI), Z6
	VMOVDQU64 (SI), Z7

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z8
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z8, Z9
	VXORPD         Z6, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z1, Z8, Z9
	VXORPD         Z7, Z9, Z7

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z8
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z8, Z9
	VXORPD         Z6, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z3, Z8, Z9
	VXORPD         Z7, Z9, Z7

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (CX), Z8
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z4, Z8, Z9
	VXORPD         Z6, Z9, Z6
	VGF2P8AFFINEQB $0x00, Z5, Z8, Z9
	VXORPD         Z7, Z9, Z7

	// Store 2 outputs
	VMOVDQU64 Z6, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z7, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x2_64Xor_loop
	VZEROUPPER

mulGFNI_3x2_64Xor_end:
	RET

// func mulAvxGFNI_3x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x2Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), CX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), SI
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI
	ADDQ R8, SI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, CX

mulAvxGFNI_3x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (DI), Y6
	VMOVDQU (SI), Y7

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y8
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y8, Y9
	VXORPD         Y6, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y8, Y9
	VXORPD         Y7, Y9, Y7

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y8
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y8, Y9
	VXORPD         Y6, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y8, Y9
	VXORPD         Y7, Y9, Y7

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (CX), Y8
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y4, Y8, Y9
	VXORPD         Y6, Y9, Y6
	VGF2P8AFFINEQB $0x00, Y5, Y8, Y9
	VXORPD         Y7, Y9, Y7

	// Store 2 outputs
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y7, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x2Xor_loop
	VZEROUPPER

mulAvxGFNI_3x2Xor_end:
	RET

// func mulAvxTwo_3x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 33 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), DI
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8
	ADDQ R9, DI

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, SI
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_3x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R8), Y0
	VMOVDQU 32(R8), Y1
	VMOVDQU (DI), Y2
	VMOVDQU 32(DI), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R8)
	VMOVDQU Y1, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_3x2_64Xor_end:
	RET

// func mulAvxTwo_3x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 46 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), DI
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DI

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_3x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R8)
	VMOVDQU Y1, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y2, (R9)
	VMOVDQU Y3, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y4, (DI)
	VMOVDQU Y5, 32(DI)
	ADDQ    $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x3_64_loop
	VZEROUPPER

mulAvxTwo_3x3_64_end:
	RET

// func mulGFNI_3x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), SI
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, SI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, CX

mulGFNI_3x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z12
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z12, Z9
	VGF2P8AFFINEQB $0x00, Z1, Z12, Z10
	VGF2P8AFFINEQB $0x00, Z2, Z12, Z11

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z12
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (CX), Z12
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Store 3 outputs
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z10, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z11, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x3_64_loop
	VZEROUPPER

mulGFNI_3x3_64_end:
	RET

// func mulAvxGFNI_3x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x3(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), CX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), SI
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, SI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, CX

mulAvxGFNI_3x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (DX), Y12
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y12, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y12, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y12, Y11

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (BX), Y12
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (CX), Y12
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Store 3 outputs
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x3_loop
	VZEROUPPER

mulAvxGFNI_3x3_end:
	RET

// func mulGFNI_3x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), SI
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, SI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, CX

mulGFNI_3x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (DI), Z9
	VMOVDQU64 (R8), Z10
	VMOVDQU64 (SI), Z11

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z12
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z12
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (CX), Z12
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
	VXORPD         Z9, Z13, Z9
	VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Store 3 outputs
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z10, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z11, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x3_64Xor_loop
	VZEROUPPER

mulGFNI_3x3_64Xor_end:
	RET

// func mulAvxGFNI_3x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x3Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), CX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), SI
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, DI
	ADDQ R9, R8
	ADDQ R9, SI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, CX

mulAvxGFNI_3x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (DI), Y9
	VMOVDQU (R8), Y10
	VMOVDQU (SI), Y11

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (DX), Y12
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (BX), Y12
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (CX), Y12
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
	VXORPD         Y9, Y13, Y9
	VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Store 3 outputs
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x3Xor_loop
	VZEROUPPER

mulAvxGFNI_3x3Xor_end:
	RET

// func mulAvxTwo_3x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 46 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), DI
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DI

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_3x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R8), Y0
	VMOVDQU 32(R8), Y1
	VMOVDQU (R9), Y2
	VMOVDQU 32(R9), Y3
	VMOVDQU (DI), Y4
	VMOVDQU 32(DI), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R8)
	VMOVDQU Y1, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y2, (R9)
	VMOVDQU Y3, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y4, (DI)
	VMOVDQU Y5, 32(DI)
	ADDQ    $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_3x3_64Xor_end:
	RET

// func mulAvxTwo_3x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x4(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 33 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), DI
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DI

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_3x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x4_loop
	VZEROUPPER

mulAvxTwo_3x4_end:
	RET

// func mulGFNI_3x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x4_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), SI
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, SI

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, CX

mulGFNI_3x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z16
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z2, Z16, Z14
	VGF2P8AFFINEQB $0x00, Z3, Z16, Z15

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z16
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
	VXORPD         Z12, Z17, Z12
	VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
	VXORPD         Z13, Z17, Z13
	VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (CX), Z16
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
	VXORPD         Z12, Z17, Z12
	VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
	VXORPD         Z13, Z17, Z13
	VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Store 4 outputs
	VMOVDQU64 Z12, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z13, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z14, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z15, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x4_64_loop
	VZEROUPPER

mulGFNI_3x4_64_end:
	RET

// func mulAvxGFNI_3x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x4(SB), $0-88
	// Loading 10 of 12 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), DI
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DI

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DX

mulAvxGFNI_3x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x4_loop
	VZEROUPPER

mulAvxGFNI_3x4_end:
	RET

// func mulGFNI_3x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x4_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), SI
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, SI

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, CX

mulGFNI_3x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (DI), Z12
	VMOVDQU64 (R8), Z13
	VMOVDQU64 (R9), Z14
	VMOVDQU64 (SI), Z15

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z16
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z16, Z17
	VXORPD         Z12, Z17, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z16, Z17
	VXORPD         Z13, Z17, Z13
	VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z16
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
	VXORPD         Z12, Z17, Z12
	VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
	VXORPD         Z13, Z17, Z13
	VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (CX), Z16
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
	VXORPD         Z12, Z17, Z12
	VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
	VXORPD         Z13, Z17, Z13
	VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Store 4 outputs
	VMOVDQU64 Z12, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z13, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z14, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z15, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x4_64Xor_loop
	VZEROUPPER

mulGFNI_3x4_64Xor_end:
	RET

// func mulAvxGFNI_3x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x4Xor(SB), $0-88
	// Loading 10 of 12 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), DI
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DI

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DX

mulAvxGFNI_3x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (R8), Y10
	VMOVDQU (R9), Y11
	VMOVDQU (R10), Y12
	VMOVDQU (DI), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x4Xor_loop
	VZEROUPPER

mulAvxGFNI_3x4Xor_end:
	RET

// func mulAvxTwo_3x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x4Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 33 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), DI
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DI

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_3x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (R8), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R9), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R10), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (DI), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x4Xor_loop
	VZEROUPPER

mulAvxTwo_3x4Xor_end:
	RET

// func mulAvxTwo_3x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x5(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 40 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), DI
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DI

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_3x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x5_loop
	VZEROUPPER

mulAvxTwo_3x5_end:
	RET

// func mulGFNI_3x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x5_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), SI
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, SI

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, CX

mulGFNI_3x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z20
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z20, Z15
	VGF2P8AFFINEQB $0x00, Z1, Z20, Z16
	VGF2P8AFFINEQB $0x00, Z2, Z20, Z17
	VGF2P8AFFINEQB $0x00, Z3, Z20, Z18
	VGF2P8AFFINEQB $0x00, Z4, Z20, Z19

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z20
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
	VXORPD         Z15, Z21, Z15
	VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (CX), Z20
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
	VXORPD         Z15, Z21, Z15
	VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Store 5 outputs
	VMOVDQU64 Z15, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z16, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z17, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x5_64_loop
	VZEROUPPER

mulGFNI_3x5_64_end:
	RET

// func mulAvxGFNI_3x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x5(SB), $0-88
	// Loading 9 of 15 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), DI
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DI

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DX

mulAvxGFNI_3x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x5_loop
	VZEROUPPER

mulAvxGFNI_3x5_end:
	RET

// func mulGFNI_3x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x5_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), SI
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, SI

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, CX

mulGFNI_3x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (DI), Z15
	VMOVDQU64 (R8), Z16
	VMOVDQU64 (R9), Z17
	VMOVDQU64 (R10), Z18
	VMOVDQU64 (SI), Z19

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z20
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
	VXORPD         Z15, Z21, Z15
	VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z20
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
	VXORPD         Z15, Z21, Z15
	VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (CX), Z20
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
	VXORPD         Z15, Z21, Z15
	VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Store 5 outputs
	VMOVDQU64 Z15, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z16, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z17, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x5_64Xor_loop
	VZEROUPPER

mulGFNI_3x5_64Xor_end:
	RET

// func mulAvxGFNI_3x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x5Xor(SB), $0-88
	// Loading 9 of 15 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), DI
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DI

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DX

mulAvxGFNI_3x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (R8), Y9
	VMOVDQU (R9), Y10
	VMOVDQU (R10), Y11
	VMOVDQU (R11), Y12
	VMOVDQU (DI), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x5Xor_loop
	VZEROUPPER

mulAvxGFNI_3x5Xor_end:
	RET

// func mulAvxTwo_3x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x5Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 40 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), DI
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DI

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_3x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (R8), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (R9), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R10), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R11), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (DI), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x5Xor_loop
	VZEROUPPER

mulAvxTwo_3x5Xor_end:
	RET

// func mulAvxTwo_3x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x6(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), DI
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DI

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_3x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x6_loop
	VZEROUPPER

mulAvxTwo_3x6_end:
	RET

// func mulGFNI_3x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x6_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), R11
	MOVQ            120(SI), SI
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, SI

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, CX

mulGFNI_3x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z19
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z20
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z21
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z22
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z23

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (BX), Z24
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 6 outputs
	VMOVDQU64 Z18, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z19, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z20, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z21, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z22, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z23, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x6_64_loop
	VZEROUPPER

mulGFNI_3x6_64_end:
	RET

// func mulAvxGFNI_3x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x6(SB), $0-88
	// Loading 8 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), DI
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DI

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DX

mulAvxGFNI_3x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x6_loop
	VZEROUPPER

mulAvxGFNI_3x6_end:
	RET

// func mulGFNI_3x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x6_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), R11
	MOVQ            120(SI), SI
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, SI

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, CX

mulGFNI_3x6_64Xor_loop:
	// Load 6 outputs
	VMOVDQU64 (DI), Z18
	VMOVDQU64 (R8), Z19
	VMOVDQU64 (R9), Z20
	VMOVDQU64 (R10), Z21
	VMOVDQU64 (R11), Z22
	VMOVDQU64 (SI), Z23

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (BX), Z24
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z18, Z25, Z18
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z19, Z25, Z19
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 6 outputs
	VMOVDQU64 Z18, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z19, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z20, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z21, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z22, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z23, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x6_64Xor_loop
	VZEROUPPER

mulGFNI_3x6_64Xor_end:
	RET

// func mulAvxGFNI_3x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x6Xor(SB), $0-88
	// Loading 8 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), DI
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DI

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DX

mulAvxGFNI_3x6Xor_loop:
	// Load 6 outputs
	VMOVDQU (R8), Y8
	VMOVDQU (R9), Y9
	VMOVDQU (R10), Y10
	VMOVDQU (R11), Y11
	VMOVDQU (R12), Y12
	VMOVDQU (DI), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x6Xor_loop
	VZEROUPPER

mulAvxGFNI_3x6Xor_end:
	RET

// func mulAvxTwo_3x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x6Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), DI
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DI

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_3x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (R8), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU (R9), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU (R10), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU (R11), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU (R12), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU (DI), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x6Xor_loop
	VZEROUPPER

mulAvxTwo_3x6Xor_end:
	RET

// func mulAvxTwo_3x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x7(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 54 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), DI
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, DI

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_3x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x7_loop
	VZEROUPPER

mulAvxTwo_3x7_end:
	RET

// func mulGFNI_3x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x7_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), R11
	MOVQ            120(SI), R12
	MOVQ            144(SI), SI
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, SI

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, CX

mulGFNI_3x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (DX), Z28
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z28, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z28, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z28, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z28, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z28, Z27

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (BX), Z28
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
	VXORPD         Z21, Z29, Z21
	VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
	VXORPD         Z22, Z29, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
	VXORPD         Z23, Z29, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (CX), Z28
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
	VXORPD         Z21, Z29, Z21
	VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
	VXORPD         Z22, Z29, Z22
	VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
	VXORPD         Z23, Z29, Z23
	VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Store 7 outputs
	VMOVDQU64 Z21, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z22, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x7_64_loop
	VZEROUPPER

mulGFNI_3x7_64_end:
	RET

// func mulAvxGFNI_3x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x7(SB), $0-88
	// Loading 7 of 21 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), DI
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, DI

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DX

mulAvxGFNI_3x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x7_loop
	VZEROUPPER

mulAvxGFNI_3x7_end:
	RET

// func mulGFNI_3x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x7_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), CX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), R11
	MOVQ            120(SI), R12
	MOVQ            144(SI), SI
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, SI

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, CX

mulGFNI_3x7_64Xor_loop:
	// Load 7 outputs
	VMOVDQU64 (DI), Z21
	VMOVDQU64 (R8), Z22
	VMOVDQU64 (R9), Z23
	VMOVDQU64 (R10), Z24
	VMOVDQU64 (R11), Z25
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (SI), Z27

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (DX), Z28
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z28, Z29
	VXORPD         Z21, Z29, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z28, Z29
	VXORPD         Z22, Z29, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z28, Z29
	VXORPD         Z23, Z29, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (BX), Z28
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
	VXORPD         Z21, Z29, Z21
	VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
	VXORPD         Z22, Z29, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
	VXORPD         Z23, Z29, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (CX), Z28
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
	VXORPD         Z21, Z29, Z21
	VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
	VXORPD         Z22, Z29, Z22
	VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
	VXORPD         Z23, Z29, Z23
	VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Store 7 outputs
	VMOVDQU64 Z21, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z22, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x7_64Xor_loop
	VZEROUPPER

mulGFNI_3x7_64Xor_end:
	RET

// func mulAvxGFNI_3x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x7Xor(SB), $0-88
	// Loading 7 of 21 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), DI
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, DI

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DX

mulAvxGFNI_3x7Xor_loop:
	// Load 7 outputs
	VMOVDQU (R8), Y7
	VMOVDQU (R9), Y8
	VMOVDQU (R10), Y9
	VMOVDQU (R11), Y10
	VMOVDQU (R12), Y11
	VMOVDQU (R13), Y12
	VMOVDQU (DI), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x7Xor_loop
	VZEROUPPER

mulAvxGFNI_3x7Xor_end:
	RET

// func mulAvxTwo_3x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x7Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 54 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), DI
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, DI

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_3x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (R8), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU (R9), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU (R10), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU (R11), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU (R12), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU (R13), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU (DI), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x7Xor_loop
	VZEROUPPER

mulAvxTwo_3x7Xor_end:
	RET

// func mulAvxTwo_3x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x8(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 61 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), R14
	MOVQ  168(DI), DI
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, DI

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_3x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x8_loop
	VZEROUPPER

mulAvxTwo_3x8_end:
	RET

// func mulGFNI_3x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x8_64(SB), $0-88
	// Loading 22 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), R13
	MOVQ            144(DI), R14
	MOVQ            168(DI), DI
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, DI

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DX

mulGFNI_3x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	VMOVDQU64 Z22, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x8_64_loop
	VZEROUPPER

mulGFNI_3x8_64_end:
	RET

// func mulAvxGFNI_3x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x8(SB), $0-88
	// Loading 6 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), R14
	MOVQ         168(DI), DI
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, DI

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DX

mulAvxGFNI_3x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x8_loop
	VZEROUPPER

mulAvxGFNI_3x8_end:
	RET

// func mulGFNI_3x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x8_64Xor(SB), $0-88
	// Loading 22 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), R13
	MOVQ            144(DI), R14
	MOVQ            168(DI), DI
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, DI

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DX

mulGFNI_3x8_64Xor_loop:
	// Load 8 outputs
	VMOVDQU64 (R8), Z22
	VMOVDQU64 (R9), Z23
	VMOVDQU64 (R10), Z24
	VMOVDQU64 (R11), Z25
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (R13), Z27
	VMOVDQU64 (R14), Z28
	VMOVDQU64 (DI), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	VMOVDQU64 Z22, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x8_64Xor_loop
	VZEROUPPER

mulGFNI_3x8_64Xor_end:
	RET

// func mulAvxGFNI_3x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x8Xor(SB), $0-88
	// Loading 6 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), R14
	MOVQ         168(DI), DI
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, DI

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DX

mulAvxGFNI_3x8Xor_loop:
	// Load 8 outputs
	VMOVDQU (R8), Y6
	VMOVDQU (R9), Y7
	VMOVDQU (R10), Y8
	VMOVDQU (R11), Y9
	VMOVDQU (R12), Y10
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (DI), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x8Xor_loop
	VZEROUPPER

mulAvxGFNI_3x8Xor_end:
	RET

// func mulAvxTwo_3x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x8Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 61 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), R14
	MOVQ  168(DI), DI
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, DI

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_3x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (R8), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU (R9), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU (R10), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU (R11), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU (R12), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU (R13), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU (R14), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU (DI), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y7, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x8Xor_loop
	VZEROUPPER

mulAvxTwo_3x8Xor_end:
	RET

// func mulAvxTwo_3x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x9(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 68 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), R14
	MOVQ  168(DI), R15
	MOVQ  192(DI), DI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_3x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y7, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x9_loop
	VZEROUPPER

mulAvxTwo_3x9_end:
	RET

// func mulGFNI_3x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x9_64(SB), $8-88
	// Loading 21 of 27 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), R13
	MOVQ            144(DI), R14
	MOVQ            168(DI), R15
	MOVQ            192(DI), DI
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DX

mulGFNI_3x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	VMOVDQU64 Z21, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x9_64_loop
	VZEROUPPER

mulGFNI_3x9_64_end:
	RET

// func mulAvxGFNI_3x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x9(SB), $8-88
	// Loading 5 of 27 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), R14
	MOVQ         168(DI), R15
	MOVQ         192(DI), DI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DX

mulAvxGFNI_3x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x9_loop
	VZEROUPPER

mulAvxGFNI_3x9_end:
	RET

// func mulGFNI_3x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x9_64Xor(SB), $8-88
	// Loading 21 of 27 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), R13
	MOVQ            144(DI), R14
	MOVQ            168(DI), R15
	MOVQ            192(DI), DI
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DX

mulGFNI_3x9_64Xor_loop:
	// Load 9 outputs
	VMOVDQU64 (R8), Z21
	VMOVDQU64 (R9), Z22
	VMOVDQU64 (R10), Z23
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (DI), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	VMOVDQU64 Z21, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_3x9_64Xor_loop
	VZEROUPPER

mulGFNI_3x9_64Xor_end:
	RET

// func mulAvxGFNI_3x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x9Xor(SB), $8-88
	// Loading 5 of 27 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), R14
	MOVQ         168(DI), R15
	MOVQ         192(DI), DI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DX

mulAvxGFNI_3x9Xor_loop:
	// Load 9 outputs
	VMOVDQU (R8), Y5
	VMOVDQU (R9), Y6
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (DI), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_3x9Xor_loop
	VZEROUPPER

mulAvxGFNI_3x9Xor_end:
	RET

// func mulAvxTwo_3x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x9Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 68 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), R14
	MOVQ  168(DI), R15
	MOVQ  192(DI), DI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_3x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (R8), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU (R9), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU (R10), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU (R11), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU (R12), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU (R13), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU (R14), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU (R15), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU (DI), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y7, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_3x9Xor_loop
	VZEROUPPER

mulAvxTwo_3x9Xor_end:
	RET

// func mulAvxTwo_3x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x10(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 75 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x10_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), AX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), R14
	MOVQ  192(SI), R15
	MOVQ  216(SI), SI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X10
	VPBROADCASTB X10, Y10
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_3x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (AX), Y13
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y8, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y9, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_3x10_loop
	VZEROUPPER

mulAvxTwo_3x10_end:
	RET

// func mulGFNI_3x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x10_64(SB), $8-88
	// Loading 20 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), AX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), R11
	MOVQ            120(SI), R12
	MOVQ            144(SI), R13
	MOVQ            168(SI), R14
	MOVQ            192(SI), R15
	MOVQ            216(SI), SI
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_3x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	VMOVDQU64 Z20, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z21, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_3x10_64_loop
	VZEROUPPER

mulGFNI_3x10_64_end:
	RET

// func mulAvxGFNI_3x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x10(SB), $8-88
	// Loading 4 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), AX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), R14
	MOVQ         192(SI), R15
	MOVQ         216(SI), SI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_3x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_3x10_loop
	VZEROUPPER

mulAvxGFNI_3x10_end:
	RET

// func mulGFNI_3x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_3x10_64Xor(SB), $8-88
	// Loading 20 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_3x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), AX
	MOVQ            out_base+48(FP), SI
	MOVQ            out_base+48(FP), SI
	MOVQ            (SI), DI
	MOVQ            24(SI), R8
	MOVQ            48(SI), R9
	MOVQ            72(SI), R10
	MOVQ            96(SI), R11
	MOVQ            120(SI), R12
	MOVQ            144(SI), R13
	MOVQ            168(SI), R14
	MOVQ            192(SI), R15
	MOVQ            216(SI), SI
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_3x10_64Xor_loop:
	// Load 10 outputs
	VMOVDQU64 (DI), Z20
	VMOVDQU64 (R8), Z21
	VMOVDQU64 (R9), Z22
	VMOVDQU64 (R10), Z23
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (SI), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	VMOVDQU64 Z20, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z21, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (SI)
	ADDQ      $0x40, SI

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_3x10_64Xor_loop
	VZEROUPPER

mulGFNI_3x10_64Xor_end:
	RET

// func mulAvxGFNI_3x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_3x10Xor(SB), $8-88
	// Loading 4 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_3x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), AX
	MOVQ         out_base+48(FP), SI
	MOVQ         out_base+48(FP), SI
	MOVQ         (SI), DI
	MOVQ         24(SI), R8
	MOVQ         48(SI), R9
	MOVQ         72(SI), R10
	MOVQ         96(SI), R11
	MOVQ         120(SI), R12
	MOVQ         144(SI), R13
	MOVQ         168(SI), R14
	MOVQ         192(SI), R15
	MOVQ         216(SI), SI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_3x10Xor_loop:
	// Load 10 outputs
	VMOVDQU (DI), Y4
	VMOVDQU (R8), Y5
	VMOVDQU (R9), Y6
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (SI), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_3x10Xor_loop
	VZEROUPPER

mulAvxGFNI_3x10Xor_end:
	RET

// func mulAvxTwo_3x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_3x10Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 75 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_3x10Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), AX
	MOVQ  out_base+48(FP), SI
	MOVQ  (SI), DI
	MOVQ  24(SI), R8
	MOVQ  48(SI), R9
	MOVQ  72(SI), R10
	MOVQ  96(SI), R11
	MOVQ  120(SI), R12
	MOVQ  144(SI), R13
	MOVQ  168(SI), R14
	MOVQ  192(SI), R15
	MOVQ  216(SI), SI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, SI

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X10
	VPBROADCASTB X10, Y10
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_3x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (DI), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU (R8), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU (R9), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU (R12), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU (R13), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU (R14), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU (R15), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU (SI), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (AX), Y13
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	VMOVDQU Y0, (DI)
	ADDQ    $0x20, DI
	VMOVDQU Y1, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y2, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y5, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y6, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y7, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y8, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y9, (SI)
	ADDQ    $0x20, SI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_3x10Xor_loop
	VZEROUPPER

mulAvxTwo_3x10Xor_end:
	RET

// func mulAvxTwo_4x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R8
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, SI
	ADDQ         R9, DI
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_4x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R8)
	VMOVDQU Y1, 32(R8)
	ADDQ    $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x1_64_loop
	VZEROUPPER

mulAvxTwo_4x1_64_end:
	RET

// func mulGFNI_4x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 7 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), DI
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, CX

mulGFNI_4x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z5
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z5, Z4

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z5
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z5
	VXORPD         Z4, Z5, Z4

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z5
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z5, Z5
	VXORPD         Z4, Z5, Z4

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (CX), Z5
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z3, Z5, Z5
	VXORPD         Z4, Z5, Z4

	// Store 1 outputs
	VMOVDQU64 Z4, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x1_64_loop
	VZEROUPPER

mulGFNI_4x1_64_end:
	RET

// func mulAvxGFNI_4x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 7 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), CX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), DI
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, CX

mulAvxGFNI_4x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y5
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y5, Y4

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y5
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y5, Y5
	VXORPD         Y4, Y5, Y4

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y5
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y5, Y5
	VXORPD         Y4, Y5, Y4

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (CX), Y5
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y3, Y5, Y5
	VXORPD         Y4, Y5, Y4

	// Store 1 outputs
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x1_loop
	VZEROUPPER

mulAvxGFNI_4x1_end:
	RET

// func mulGFNI_4x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 7 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), DI
	MOVQ            start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, CX

mulGFNI_4x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (DI), Z4

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z5
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z5, Z5
	VXORPD         Z4, Z5, Z4

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z5
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z5
	VXORPD         Z4, Z5, Z4

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z5
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z5, Z5
	VXORPD         Z4, Z5, Z4

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (CX), Z5
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z3, Z5, Z5
	VXORPD         Z4, Z5, Z4

	// Store 1 outputs
	VMOVDQU64 Z4, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x1_64Xor_loop
	VZEROUPPER

mulGFNI_4x1_64Xor_end:
	RET

// func mulAvxGFNI_4x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 7 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), CX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), DI
	MOVQ         start+72(FP), R8

	// Add start offset to output
	ADDQ R8, DI

	// Add start offset to input
	ADDQ R8, DX
	ADDQ R8, BX
	ADDQ R8, SI
	ADDQ R8, CX

mulAvxGFNI_4x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (DI), Y4

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y5
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y5, Y5
	VXORPD         Y4, Y5, Y4

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y5
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y5, Y5
	VXORPD         Y4, Y5, Y4

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y5
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y5, Y5
	VXORPD         Y4, Y5, Y4

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (CX), Y5
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y3, Y5, Y5
	VXORPD         Y4, Y5, Y4

	// Store 1 outputs
	VMOVDQU Y4, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x1Xor_loop
	VZEROUPPER

mulAvxGFNI_4x1Xor_end:
	RET

// func mulAvxTwo_4x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R8
	MOVQ  start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, SI
	ADDQ         R9, DI
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R9
	MOVQ         R9, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_4x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (R8), Y0
	VMOVDQU 32(R8), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R8)
	VMOVDQU Y1, 32(R8)
	ADDQ    $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_4x1_64Xor_end:
	RET

// func mulAvxTwo_4x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 41 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R8
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9
	ADDQ R10, R8

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_4x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R9)
	VMOVDQU Y1, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y2, (R8)
	VMOVDQU Y3, 32(R8)
	ADDQ    $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x2_64_loop
	VZEROUPPER

mulAvxTwo_4x2_64_end:
	RET

// func mulGFNI_4x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), DI
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8
	ADDQ R9, DI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, CX

mulGFNI_4x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z10
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z10, Z8
	VGF2P8AFFINEQB $0x00, Z1, Z10, Z9

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z10
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z3, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z10
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z5, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (CX), Z10
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z7, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Store 2 outputs
	VMOVDQU64 Z8, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x2_64_loop
	VZEROUPPER

mulGFNI_4x2_64_end:
	RET

// func mulAvxGFNI_4x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x2(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), CX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), DI
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8
	ADDQ R9, DI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, CX

mulAvxGFNI_4x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y10
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y10, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y10, Y9

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y10
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (SI), Y10
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y5, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (CX), Y10
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y6, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Store 2 outputs
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x2_loop
	VZEROUPPER

mulAvxGFNI_4x2_end:
	RET

// func mulGFNI_4x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), DI
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8
	ADDQ R9, DI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, CX

mulGFNI_4x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (R8), Z8
	VMOVDQU64 (DI), Z9

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z10
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z1, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z10
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z3, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z10
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z5, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (CX), Z10
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z10, Z11
	VXORPD         Z8, Z11, Z8
	VGF2P8AFFINEQB $0x00, Z7, Z10, Z11
	VXORPD         Z9, Z11, Z9

	// Store 2 outputs
	VMOVDQU64 Z8, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z9, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x2_64Xor_loop
	VZEROUPPER

mulGFNI_4x2_64Xor_end:
	RET

// func mulAvxGFNI_4x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x2Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), CX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), DI
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8
	ADDQ R9, DI

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, CX

mulAvxGFNI_4x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (R8), Y8
	VMOVDQU (DI), Y9

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y10
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y10
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (SI), Y10
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y5, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (CX), Y10
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y6, Y10, Y11
	VXORPD         Y8, Y11, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y10, Y11
	VXORPD         Y9, Y11, Y9

	// Store 2 outputs
	VMOVDQU Y8, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y9, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x2Xor_loop
	VZEROUPPER

mulAvxGFNI_4x2Xor_end:
	RET

// func mulAvxTwo_4x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 41 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R8
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9
	ADDQ R10, R8

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_4x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R9), Y0
	VMOVDQU 32(R9), Y1
	VMOVDQU (R8), Y2
	VMOVDQU 32(R8), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R9)
	VMOVDQU Y1, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y2, (R8)
	VMOVDQU Y3, 32(R8)
	ADDQ    $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_4x2_64Xor_end:
	RET

// func mulAvxTwo_4x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 58 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R8
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, R8

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_4x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R9)
	VMOVDQU Y1, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y2, (R10)
	VMOVDQU Y3, 32(R10)
	ADDQ    $0x40, R10
	VMOVDQU Y4, (R8)
	VMOVDQU Y5, 32(R8)
	ADDQ    $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x3_64_loop
	VZEROUPPER

mulAvxTwo_4x3_64_end:
	RET

// func mulGFNI_4x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), DI
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DI

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, CX

mulGFNI_4x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z15
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z15, Z13
	VGF2P8AFFINEQB $0x00, Z2, Z15, Z14

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z15
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z4, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z15
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (CX), Z15
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z10, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z11, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Store 3 outputs
	VMOVDQU64 Z12, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z14, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x3_64_loop
	VZEROUPPER

mulGFNI_4x3_64_end:
	RET

// func mulAvxGFNI_4x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x3(SB), $0-88
	// Loading 11 of 12 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R8
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, R8

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, DX

mulAvxGFNI_4x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x3_loop
	VZEROUPPER

mulAvxGFNI_4x3_end:
	RET

// func mulGFNI_4x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), DI
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R8
	ADDQ R10, R9
	ADDQ R10, DI

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, CX

mulGFNI_4x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (R8), Z12
	VMOVDQU64 (R9), Z13
	VMOVDQU64 (DI), Z14

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z15
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z2, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z15
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z4, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z15
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (CX), Z15
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
	VXORPD         Z12, Z16, Z12
	VGF2P8AFFINEQB $0x00, Z10, Z15, Z16
	VXORPD         Z13, Z16, Z13
	VGF2P8AFFINEQB $0x00, Z11, Z15, Z16
	VXORPD         Z14, Z16, Z14

	// Store 3 outputs
	VMOVDQU64 Z12, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z14, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x3_64Xor_loop
	VZEROUPPER

mulGFNI_4x3_64Xor_end:
	RET

// func mulAvxGFNI_4x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x3Xor(SB), $0-88
	// Loading 11 of 12 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 17 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R8
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, R8

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, DX

mulAvxGFNI_4x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU (R10), Y12
	VMOVDQU (R8), Y13

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x3Xor_loop
	VZEROUPPER

mulAvxGFNI_4x3Xor_end:
	RET

// func mulAvxTwo_4x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 58 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R8
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, R8

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_4x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R9), Y0
	VMOVDQU 32(R9), Y1
	VMOVDQU (R10), Y2
	VMOVDQU 32(R10), Y3
	VMOVDQU (R8), Y4
	VMOVDQU 32(R8), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R9)
	VMOVDQU Y1, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y2, (R10)
	VMOVDQU Y3, 32(R10)
	ADDQ    $0x40, R10
	VMOVDQU Y4, (R8)
	VMOVDQU Y5, 32(R8)
	ADDQ    $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_4x3_64Xor_end:
	RET

// func mulAvxTwo_4x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x4(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 41 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R8
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R8

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_4x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x4_loop
	VZEROUPPER

mulAvxTwo_4x4_end:
	RET

// func mulGFNI_4x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x4_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), DI
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DI

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, CX

mulGFNI_4x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z20
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z20, Z16
	VGF2P8AFFINEQB $0x00, Z1, Z20, Z17
	VGF2P8AFFINEQB $0x00, Z2, Z20, Z18
	VGF2P8AFFINEQB $0x00, Z3, Z20, Z19

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z20
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z20
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (CX), Z20
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Store 4 outputs
	VMOVDQU64 Z16, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z17, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x4_64_loop
	VZEROUPPER

mulGFNI_4x4_64_end:
	RET

// func mulAvxGFNI_4x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x4(SB), $0-88
	// Loading 10 of 16 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R8
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R8

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, DX

mulAvxGFNI_4x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x4_loop
	VZEROUPPER

mulAvxGFNI_4x4_end:
	RET

// func mulGFNI_4x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x4_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), DI
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, DI

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, CX

mulGFNI_4x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (R8), Z16
	VMOVDQU64 (R9), Z17
	VMOVDQU64 (R10), Z18
	VMOVDQU64 (DI), Z19

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z20
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z20
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z20
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (CX), Z20
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
	VXORPD         Z16, Z21, Z16
	VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
	VXORPD         Z17, Z21, Z17
	VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Store 4 outputs
	VMOVDQU64 Z16, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z17, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x4_64Xor_loop
	VZEROUPPER

mulGFNI_4x4_64Xor_end:
	RET

// func mulAvxGFNI_4x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x4Xor(SB), $0-88
	// Loading 10 of 16 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R8
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R8

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, DX

mulAvxGFNI_4x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (R9), Y10
	VMOVDQU (R10), Y11
	VMOVDQU (R11), Y12
	VMOVDQU (R8), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x4Xor_loop
	VZEROUPPER

mulAvxGFNI_4x4Xor_end:
	RET

// func mulAvxTwo_4x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x4Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 41 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R8
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R8

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_4x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (R9), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R10), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R11), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (R8), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x4Xor_loop
	VZEROUPPER

mulAvxTwo_4x4Xor_end:
	RET

// func mulAvxTwo_4x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x5(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R8
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R8

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_4x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x5_loop
	VZEROUPPER

mulAvxTwo_4x5_end:
	RET

// func mulGFNI_4x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x5_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 27 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), DI
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DI

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, CX

mulGFNI_4x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z25
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z25, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z25, Z24

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z25
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z25, Z26
	VXORPD         Z20, Z26, Z20
	VGF2P8AFFINEQB $0x00, Z6, Z25, Z26
	VXORPD         Z21, Z26, Z21
	VGF2P8AFFINEQB $0x00, Z7, Z25, Z26
	VXORPD         Z22, Z26, Z22
	VGF2P8AFFINEQB $0x00, Z8, Z25, Z26
	VXORPD         Z23, Z26, Z23
	VGF2P8AFFINEQB $0x00, Z9, Z25, Z26
	VXORPD         Z24, Z26, Z24

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (SI), Z25
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z25, Z26
	VXORPD         Z20, Z26, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z25, Z26
	VXORPD         Z21, Z26, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z25, Z26
	VXORPD         Z22, Z26, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z25, Z26
	VXORPD         Z23, Z26, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z25, Z26
	VXORPD         Z24, Z26, Z24

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (CX), Z25
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z15, Z25, Z26
	VXORPD         Z20, Z26, Z20
	VGF2P8AFFINEQB $0x00, Z16, Z25, Z26
	VXORPD         Z21, Z26, Z21
	VGF2P8AFFINEQB $0x00, Z17, Z25, Z26
	VXORPD         Z22, Z26, Z22
	VGF2P8AFFINEQB $0x00, Z18, Z25, Z26
	VXORPD         Z23, Z26, Z23
	VGF2P8AFFINEQB $0x00, Z19, Z25, Z26
	VXORPD         Z24, Z26, Z24

	// Store 5 outputs
	VMOVDQU64 Z20, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z21, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z22, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z23, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z24, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x5_64_loop
	VZEROUPPER

mulGFNI_4x5_64_end:
	RET

// func mulAvxGFNI_4x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x5(SB), $0-88
	// Loading 9 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 27 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R8
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R8

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, DX

mulAvxGFNI_4x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x5_loop
	VZEROUPPER

mulAvxGFNI_4x5_end:
	RET

// func mulGFNI_4x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x5_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 27 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), DI
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, DI

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, CX

mulGFNI_4x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (R8), Z20
	VMOVDQU64 (R9), Z21
	VMOVDQU64 (R10), Z22
	VMOVDQU64 (R11), Z23
	VMOVDQU64 (DI), Z24

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z25
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z25, Z26
	VXORPD         Z20, Z26, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z25, Z26
	VXORPD         Z21, Z26, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z25, Z26
	VXORPD         Z22, Z26, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z25, Z26
	VXORPD         Z23, Z26, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z25, Z26
	VXORPD         Z24, Z26, Z24

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z25
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z25, Z26
	VXORPD         Z20, Z26, Z20
	VGF2P8AFFINEQB $0x00, Z6, Z25, Z26
	VXORPD         Z21, Z26, Z21
	VGF2P8AFFINEQB $0x00, Z7, Z25, Z26
	VXORPD         Z22, Z26, Z22
	VGF2P8AFFINEQB $0x00, Z8, Z25, Z26
	VXORPD         Z23, Z26, Z23
	VGF2P8AFFINEQB $0x00, Z9, Z25, Z26
	VXORPD         Z24, Z26, Z24

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (SI), Z25
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z25, Z26
	VXORPD         Z20, Z26, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z25, Z26
	VXORPD         Z21, Z26, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z25, Z26
	VXORPD         Z22, Z26, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z25, Z26
	VXORPD         Z23, Z26, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z25, Z26
	VXORPD         Z24, Z26, Z24

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (CX), Z25
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z15, Z25, Z26
	VXORPD         Z20, Z26, Z20
	VGF2P8AFFINEQB $0x00, Z16, Z25, Z26
	VXORPD         Z21, Z26, Z21
	VGF2P8AFFINEQB $0x00, Z17, Z25, Z26
	VXORPD         Z22, Z26, Z22
	VGF2P8AFFINEQB $0x00, Z18, Z25, Z26
	VXORPD         Z23, Z26, Z23
	VGF2P8AFFINEQB $0x00, Z19, Z25, Z26
	VXORPD         Z24, Z26, Z24

	// Store 5 outputs
	VMOVDQU64 Z20, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z21, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z22, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z23, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z24, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x5_64Xor_loop
	VZEROUPPER

mulGFNI_4x5_64Xor_end:
	RET

// func mulAvxGFNI_4x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x5Xor(SB), $0-88
	// Loading 9 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 27 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R8
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R8

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, DX

mulAvxGFNI_4x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (R9), Y9
	VMOVDQU (R10), Y10
	VMOVDQU (R11), Y11
	VMOVDQU (R12), Y12
	VMOVDQU (R8), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x5Xor_loop
	VZEROUPPER

mulAvxGFNI_4x5Xor_end:
	RET

// func mulAvxTwo_4x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x5Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R8
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R8

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_4x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (R9), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (R10), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R11), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R12), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (R8), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x5Xor_loop
	VZEROUPPER

mulAvxTwo_4x5Xor_end:
	RET

// func mulAvxTwo_4x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x6(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 59 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R8
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R8

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_4x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x6_loop
	VZEROUPPER

mulAvxTwo_4x6_end:
	RET

// func mulGFNI_4x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x6_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), DI
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DI

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, CX

mulGFNI_4x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z25, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z26, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z27, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z28, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x6_64_loop
	VZEROUPPER

mulGFNI_4x6_64_end:
	RET

// func mulAvxGFNI_4x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x6(SB), $0-88
	// Loading 8 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R8
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R8

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, DX

mulAvxGFNI_4x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x6_loop
	VZEROUPPER

mulAvxGFNI_4x6_end:
	RET

// func mulGFNI_4x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x6_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), CX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), DI
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, DI

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, CX

mulGFNI_4x6_64Xor_loop:
	// Load 6 outputs
	VMOVDQU64 (R8), Z24
	VMOVDQU64 (R9), Z25
	VMOVDQU64 (R10), Z26
	VMOVDQU64 (R11), Z27
	VMOVDQU64 (R12), Z28
	VMOVDQU64 (DI), Z29

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z25, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z26, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z27, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z28, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x6_64Xor_loop
	VZEROUPPER

mulGFNI_4x6_64Xor_end:
	RET

// func mulAvxGFNI_4x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x6Xor(SB), $0-88
	// Loading 8 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R8
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R8

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, DX

mulAvxGFNI_4x6Xor_loop:
	// Load 6 outputs
	VMOVDQU (R9), Y8
	VMOVDQU (R10), Y9
	VMOVDQU (R11), Y10
	VMOVDQU (R12), Y11
	VMOVDQU (R13), Y12
	VMOVDQU (R8), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x6Xor_loop
	VZEROUPPER

mulAvxGFNI_4x6Xor_end:
	RET

// func mulAvxTwo_4x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x6Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 59 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R8
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R8

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_4x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (R9), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU (R10), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU (R11), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU (R12), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU (R13), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU (R8), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x6Xor_loop
	VZEROUPPER

mulAvxTwo_4x6Xor_end:
	RET

// func mulAvxTwo_4x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x7(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 68 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R14
	MOVQ  144(R8), R8
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R8

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_4x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x7_loop
	VZEROUPPER

mulAvxTwo_4x7_end:
	RET

// func mulGFNI_4x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x7_64(SB), $0-88
	// Loading 23 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R13
	MOVQ            120(R8), R14
	MOVQ            144(R8), R8
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R8

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, DX

mulGFNI_4x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x7_64_loop
	VZEROUPPER

mulGFNI_4x7_64_end:
	RET

// func mulAvxGFNI_4x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x7(SB), $0-88
	// Loading 7 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R14
	MOVQ         144(R8), R8
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R8

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, DX

mulAvxGFNI_4x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x7_loop
	VZEROUPPER

mulAvxGFNI_4x7_end:
	RET

// func mulGFNI_4x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x7_64Xor(SB), $0-88
	// Loading 23 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R13
	MOVQ            120(R8), R14
	MOVQ            144(R8), R8
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R8

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, DX

mulGFNI_4x7_64Xor_loop:
	// Load 7 outputs
	VMOVDQU64 (R9), Z23
	VMOVDQU64 (R10), Z24
	VMOVDQU64 (R11), Z25
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (R13), Z27
	VMOVDQU64 (R14), Z28
	VMOVDQU64 (R8), Z29

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	VMOVDQU64 Z23, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x7_64Xor_loop
	VZEROUPPER

mulGFNI_4x7_64Xor_end:
	RET

// func mulAvxGFNI_4x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x7Xor(SB), $0-88
	// Loading 7 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R14
	MOVQ         144(R8), R8
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R8

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, DX

mulAvxGFNI_4x7Xor_loop:
	// Load 7 outputs
	VMOVDQU (R9), Y7
	VMOVDQU (R10), Y8
	VMOVDQU (R11), Y9
	VMOVDQU (R12), Y10
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (R8), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x7Xor_loop
	VZEROUPPER

mulAvxGFNI_4x7Xor_end:
	RET

// func mulAvxTwo_4x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x7Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 68 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R14
	MOVQ  144(R8), R8
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R8

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_4x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (R9), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU (R10), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU (R11), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU (R12), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU (R13), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU (R14), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU (R8), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y6, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x7Xor_loop
	VZEROUPPER

mulAvxTwo_4x7Xor_end:
	RET

// func mulAvxTwo_4x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x8(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 77 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R14
	MOVQ  144(R8), R15
	MOVQ  168(R8), R8
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_4x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y6, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x8_loop
	VZEROUPPER

mulAvxTwo_4x8_end:
	RET

// func mulGFNI_4x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x8_64(SB), $8-88
	// Loading 22 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R13
	MOVQ            120(R8), R14
	MOVQ            144(R8), R15
	MOVQ            168(R8), R8
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, DX

mulGFNI_4x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x8_64_loop
	VZEROUPPER

mulGFNI_4x8_64_end:
	RET

// func mulAvxGFNI_4x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x8(SB), $8-88
	// Loading 6 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R14
	MOVQ         144(R8), R15
	MOVQ         168(R8), R8
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, DX

mulAvxGFNI_4x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x8_loop
	VZEROUPPER

mulAvxGFNI_4x8_end:
	RET

// func mulGFNI_4x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x8_64Xor(SB), $8-88
	// Loading 22 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R13
	MOVQ            120(R8), R14
	MOVQ            144(R8), R15
	MOVQ            168(R8), R8
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, DX

mulGFNI_4x8_64Xor_loop:
	// Load 8 outputs
	VMOVDQU64 (R9), Z22
	VMOVDQU64 (R10), Z23
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R8), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_4x8_64Xor_loop
	VZEROUPPER

mulGFNI_4x8_64Xor_end:
	RET

// func mulAvxGFNI_4x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x8Xor(SB), $8-88
	// Loading 6 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R14
	MOVQ         144(R8), R15
	MOVQ         168(R8), R8
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, DX

mulAvxGFNI_4x8Xor_loop:
	// Load 8 outputs
	VMOVDQU (R9), Y6
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R8), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_4x8Xor_loop
	VZEROUPPER

mulAvxGFNI_4x8Xor_end:
	RET

// func mulAvxTwo_4x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x8Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 77 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R14
	MOVQ  144(R8), R15
	MOVQ  168(R8), R8
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_4x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (R9), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU (R10), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU (R11), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU (R12), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU (R13), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU (R14), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU (R15), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU (R8), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y6, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_4x8Xor_loop
	VZEROUPPER

mulAvxTwo_4x8Xor_end:
	RET

// func mulAvxTwo_4x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x9(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 86 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x9_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), AX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), R14
	MOVQ  168(DI), R15
	MOVQ  192(DI), DI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X9
	VPBROADCASTB X9, Y9
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_4x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (AX), Y12
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y7, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_4x9_loop
	VZEROUPPER

mulAvxTwo_4x9_end:
	RET

// func mulGFNI_4x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x9_64(SB), $8-88
	// Loading 21 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), AX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), R13
	MOVQ            144(DI), R14
	MOVQ            168(DI), R15
	MOVQ            192(DI), DI
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_4x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (SI), Z30
	ADDQ                $0x40, SI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	VMOVDQU64 Z21, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_4x9_64_loop
	VZEROUPPER

mulGFNI_4x9_64_end:
	RET

// func mulAvxGFNI_4x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x9(SB), $8-88
	// Loading 5 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), AX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), R14
	MOVQ         168(DI), R15
	MOVQ         192(DI), DI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_4x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_4x9_loop
	VZEROUPPER

mulAvxGFNI_4x9_end:
	RET

// func mulGFNI_4x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x9_64Xor(SB), $8-88
	// Loading 21 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), AX
	MOVQ            out_base+48(FP), DI
	MOVQ            out_base+48(FP), DI
	MOVQ            (DI), R8
	MOVQ            24(DI), R9
	MOVQ            48(DI), R10
	MOVQ            72(DI), R11
	MOVQ            96(DI), R12
	MOVQ            120(DI), R13
	MOVQ            144(DI), R14
	MOVQ            168(DI), R15
	MOVQ            192(DI), DI
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_4x9_64Xor_loop:
	// Load 9 outputs
	VMOVDQU64 (R8), Z21
	VMOVDQU64 (R9), Z22
	VMOVDQU64 (R10), Z23
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (DI), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (SI), Z30
	ADDQ                $0x40, SI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	VMOVDQU64 Z21, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (DI)
	ADDQ      $0x40, DI

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_4x9_64Xor_loop
	VZEROUPPER

mulGFNI_4x9_64Xor_end:
	RET

// func mulAvxGFNI_4x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x9Xor(SB), $8-88
	// Loading 5 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), AX
	MOVQ         out_base+48(FP), DI
	MOVQ         out_base+48(FP), DI
	MOVQ         (DI), R8
	MOVQ         24(DI), R9
	MOVQ         48(DI), R10
	MOVQ         72(DI), R11
	MOVQ         96(DI), R12
	MOVQ         120(DI), R13
	MOVQ         144(DI), R14
	MOVQ         168(DI), R15
	MOVQ         192(DI), DI
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_4x9Xor_loop:
	// Load 9 outputs
	VMOVDQU (R8), Y5
	VMOVDQU (R9), Y6
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (DI), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_4x9Xor_loop
	VZEROUPPER

mulAvxGFNI_4x9Xor_end:
	RET

// func mulAvxTwo_4x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x9Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 86 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x9Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), AX
	MOVQ  out_base+48(FP), DI
	MOVQ  (DI), R8
	MOVQ  24(DI), R9
	MOVQ  48(DI), R10
	MOVQ  72(DI), R11
	MOVQ  96(DI), R12
	MOVQ  120(DI), R13
	MOVQ  144(DI), R14
	MOVQ  168(DI), R15
	MOVQ  192(DI), DI
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, DI

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X9
	VPBROADCASTB X9, Y9
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_4x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (R8), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU (R9), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU (R10), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU (R11), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU (R12), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU (R13), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU (R14), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU (R15), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU (DI), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (AX), Y12
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	VMOVDQU Y0, (R8)
	ADDQ    $0x20, R8
	VMOVDQU Y1, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y2, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y4, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y5, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y6, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y7, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y8, (DI)
	ADDQ    $0x20, DI

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_4x9Xor_loop
	VZEROUPPER

mulAvxTwo_4x9Xor_end:
	RET

// func mulAvxTwo_4x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x10(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 95 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  start+72(FP), R9

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, SI
	ADDQ         R9, DI
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_4x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R8), R10
	VMOVDQU Y0, (R10)(R9*1)
	MOVQ    24(R8), R10
	VMOVDQU Y1, (R10)(R9*1)
	MOVQ    48(R8), R10
	VMOVDQU Y2, (R10)(R9*1)
	MOVQ    72(R8), R10
	VMOVDQU Y3, (R10)(R9*1)
	MOVQ    96(R8), R10
	VMOVDQU Y4, (R10)(R9*1)
	MOVQ    120(R8), R10
	VMOVDQU Y5, (R10)(R9*1)
	MOVQ    144(R8), R10
	VMOVDQU Y6, (R10)(R9*1)
	MOVQ    168(R8), R10
	VMOVDQU Y7, (R10)(R9*1)
	MOVQ    192(R8), R10
	VMOVDQU Y8, (R10)(R9*1)
	MOVQ    216(R8), R10
	VMOVDQU Y9, (R10)(R9*1)

	// Prepare for next loop
	ADDQ $0x20, R9
	DECQ AX
	JNZ  mulAvxTwo_4x10_loop
	VZEROUPPER

mulAvxTwo_4x10_end:
	RET

// func mulGFNI_4x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x10_64(SB), $0-88
	// Loading 20 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            start+72(FP), R9

	// Add start offset to input
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, DX

mulGFNI_4x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R8), R10
	VMOVDQU64 Z20, (R10)(R9*1)
	MOVQ      24(R8), R10
	VMOVDQU64 Z21, (R10)(R9*1)
	MOVQ      48(R8), R10
	VMOVDQU64 Z22, (R10)(R9*1)
	MOVQ      72(R8), R10
	VMOVDQU64 Z23, (R10)(R9*1)
	MOVQ      96(R8), R10
	VMOVDQU64 Z24, (R10)(R9*1)
	MOVQ      120(R8), R10
	VMOVDQU64 Z25, (R10)(R9*1)
	MOVQ      144(R8), R10
	VMOVDQU64 Z26, (R10)(R9*1)
	MOVQ      168(R8), R10
	VMOVDQU64 Z27, (R10)(R9*1)
	MOVQ      192(R8), R10
	VMOVDQU64 Z28, (R10)(R9*1)
	MOVQ      216(R8), R10
	VMOVDQU64 Z29, (R10)(R9*1)

	// Prepare for next loop
	ADDQ $0x40, R9
	DECQ AX
	JNZ  mulGFNI_4x10_64_loop
	VZEROUPPER

mulGFNI_4x10_64_end:
	RET

// func mulAvxGFNI_4x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x10(SB), $0-88
	// Loading 4 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         start+72(FP), R9

	// Add start offset to input
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, DX

mulAvxGFNI_4x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R8), R10
	VMOVDQU Y4, (R10)(R9*1)
	MOVQ    24(R8), R10
	VMOVDQU Y5, (R10)(R9*1)
	MOVQ    48(R8), R10
	VMOVDQU Y6, (R10)(R9*1)
	MOVQ    72(R8), R10
	VMOVDQU Y7, (R10)(R9*1)
	MOVQ    96(R8), R10
	VMOVDQU Y8, (R10)(R9*1)
	MOVQ    120(R8), R10
	VMOVDQU Y9, (R10)(R9*1)
	MOVQ    144(R8), R10
	VMOVDQU Y10, (R10)(R9*1)
	MOVQ    168(R8), R10
	VMOVDQU Y11, (R10)(R9*1)
	MOVQ    192(R8), R10
	VMOVDQU Y12, (R10)(R9*1)
	MOVQ    216(R8), R10
	VMOVDQU Y13, (R10)(R9*1)

	// Prepare for next loop
	ADDQ $0x20, R9
	DECQ AX
	JNZ  mulAvxGFNI_4x10_loop
	VZEROUPPER

mulAvxGFNI_4x10_end:
	RET

// func mulGFNI_4x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_4x10_64Xor(SB), $0-88
	// Loading 20 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_4x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), DX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            start+72(FP), R9

	// Add start offset to input
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, DX

mulGFNI_4x10_64Xor_loop:
	// Load 10 outputs
	MOVQ      (R8), R10
	VMOVDQU64 (R10)(R9*1), Z20
	MOVQ      24(R8), R10
	VMOVDQU64 (R10)(R9*1), Z21
	MOVQ      48(R8), R10
	VMOVDQU64 (R10)(R9*1), Z22
	MOVQ      72(R8), R10
	VMOVDQU64 (R10)(R9*1), Z23
	MOVQ      96(R8), R10
	VMOVDQU64 (R10)(R9*1), Z24
	MOVQ      120(R8), R10
	VMOVDQU64 (R10)(R9*1), Z25
	MOVQ      144(R8), R10
	VMOVDQU64 (R10)(R9*1), Z26
	MOVQ      168(R8), R10
	VMOVDQU64 (R10)(R9*1), Z27
	MOVQ      192(R8), R10
	VMOVDQU64 (R10)(R9*1), Z28
	MOVQ      216(R8), R10
	VMOVDQU64 (R10)(R9*1), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R8), R10
	VMOVDQU64 Z20, (R10)(R9*1)
	MOVQ      24(R8), R10
	VMOVDQU64 Z21, (R10)(R9*1)
	MOVQ      48(R8), R10
	VMOVDQU64 Z22, (R10)(R9*1)
	MOVQ      72(R8), R10
	VMOVDQU64 Z23, (R10)(R9*1)
	MOVQ      96(R8), R10
	VMOVDQU64 Z24, (R10)(R9*1)
	MOVQ      120(R8), R10
	VMOVDQU64 Z25, (R10)(R9*1)
	MOVQ      144(R8), R10
	VMOVDQU64 Z26, (R10)(R9*1)
	MOVQ      168(R8), R10
	VMOVDQU64 Z27, (R10)(R9*1)
	MOVQ      192(R8), R10
	VMOVDQU64 Z28, (R10)(R9*1)
	MOVQ      216(R8), R10
	VMOVDQU64 Z29, (R10)(R9*1)

	// Prepare for next loop
	ADDQ $0x40, R9
	DECQ AX
	JNZ  mulGFNI_4x10_64Xor_loop
	VZEROUPPER

mulGFNI_4x10_64Xor_end:
	RET

// func mulAvxGFNI_4x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_4x10Xor(SB), $0-88
	// Loading 4 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_4x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), DX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         start+72(FP), R9

	// Add start offset to input
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, DX

mulAvxGFNI_4x10Xor_loop:
	// Load 10 outputs
	MOVQ    (R8), R10
	VMOVDQU (R10)(R9*1), Y4
	MOVQ    24(R8), R10
	VMOVDQU (R10)(R9*1), Y5
	MOVQ    48(R8), R10
	VMOVDQU (R10)(R9*1), Y6
	MOVQ    72(R8), R10
	VMOVDQU (R10)(R9*1), Y7
	MOVQ    96(R8), R10
	VMOVDQU (R10)(R9*1), Y8
	MOVQ    120(R8), R10
	VMOVDQU (R10)(R9*1), Y9
	MOVQ    144(R8), R10
	VMOVDQU (R10)(R9*1), Y10
	MOVQ    168(R8), R10
	VMOVDQU (R10)(R9*1), Y11
	MOVQ    192(R8), R10
	VMOVDQU (R10)(R9*1), Y12
	MOVQ    216(R8), R10
	VMOVDQU (R10)(R9*1), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R8), R10
	VMOVDQU Y4, (R10)(R9*1)
	MOVQ    24(R8), R10
	VMOVDQU Y5, (R10)(R9*1)
	MOVQ    48(R8), R10
	VMOVDQU Y6, (R10)(R9*1)
	MOVQ    72(R8), R10
	VMOVDQU Y7, (R10)(R9*1)
	MOVQ    96(R8), R10
	VMOVDQU Y8, (R10)(R9*1)
	MOVQ    120(R8), R10
	VMOVDQU Y9, (R10)(R9*1)
	MOVQ    144(R8), R10
	VMOVDQU Y10, (R10)(R9*1)
	MOVQ    168(R8), R10
	VMOVDQU Y11, (R10)(R9*1)
	MOVQ    192(R8), R10
	VMOVDQU Y12, (R10)(R9*1)
	MOVQ    216(R8), R10
	VMOVDQU Y13, (R10)(R9*1)

	// Prepare for next loop
	ADDQ $0x20, R9
	DECQ AX
	JNZ  mulAvxGFNI_4x10Xor_loop
	VZEROUPPER

mulAvxGFNI_4x10Xor_end:
	RET

// func mulAvxTwo_4x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_4x10Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 95 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_4x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), DX
	MOVQ  out_base+48(FP), R8
	MOVQ  start+72(FP), R9

	// Add start offset to input
	ADDQ         R9, BX
	ADDQ         R9, SI
	ADDQ         R9, DI
	ADDQ         R9, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_4x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	MOVQ    (R8), R10
	VMOVDQU (R10)(R9*1), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	MOVQ    24(R8), R10
	VMOVDQU (R10)(R9*1), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	MOVQ    48(R8), R10
	VMOVDQU (R10)(R9*1), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	MOVQ    72(R8), R10
	VMOVDQU (R10)(R9*1), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	MOVQ    96(R8), R10
	VMOVDQU (R10)(R9*1), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	MOVQ    120(R8), R10
	VMOVDQU (R10)(R9*1), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	MOVQ    144(R8), R10
	VMOVDQU (R10)(R9*1), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	MOVQ    168(R8), R10
	VMOVDQU (R10)(R9*1), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	MOVQ    192(R8), R10
	VMOVDQU (R10)(R9*1), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	MOVQ    216(R8), R10
	VMOVDQU (R10)(R9*1), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R8), R10
	VMOVDQU Y0, (R10)(R9*1)
	MOVQ    24(R8), R10
	VMOVDQU Y1, (R10)(R9*1)
	MOVQ    48(R8), R10
	VMOVDQU Y2, (R10)(R9*1)
	MOVQ    72(R8), R10
	VMOVDQU Y3, (R10)(R9*1)
	MOVQ    96(R8), R10
	VMOVDQU Y4, (R10)(R9*1)
	MOVQ    120(R8), R10
	VMOVDQU Y5, (R10)(R9*1)
	MOVQ    144(R8), R10
	VMOVDQU Y6, (R10)(R9*1)
	MOVQ    168(R8), R10
	VMOVDQU Y7, (R10)(R9*1)
	MOVQ    192(R8), R10
	VMOVDQU Y8, (R10)(R9*1)
	MOVQ    216(R8), R10
	VMOVDQU Y9, (R10)(R9*1)

	// Prepare for next loop
	ADDQ $0x20, R9
	DECQ AX
	JNZ  mulAvxTwo_4x10Xor_loop
	VZEROUPPER

mulAvxTwo_4x10Xor_end:
	RET

// func mulAvxTwo_5x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R9
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, R8
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_5x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R9)
	VMOVDQU Y1, 32(R9)
	ADDQ    $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x1_64_loop
	VZEROUPPER

mulAvxTwo_5x1_64_end:
	RET

// func mulGFNI_5x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R8
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, CX

mulGFNI_5x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z6
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z6, Z5

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z6
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z6
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z6
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (CX), Z6
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z4, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Store 1 outputs
	VMOVDQU64 Z5, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x1_64_loop
	VZEROUPPER

mulGFNI_5x1_64_end:
	RET

// func mulAvxGFNI_5x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), CX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R8
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, CX

mulAvxGFNI_5x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y6
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y6, Y5

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y6
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y6
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y6
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (CX), Y6
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y4, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Store 1 outputs
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x1_loop
	VZEROUPPER

mulAvxGFNI_5x1_end:
	RET

// func mulGFNI_5x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R8
	MOVQ            start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, CX

mulGFNI_5x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (R8), Z5

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z6
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z6
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z6
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z6
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (CX), Z6
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z4, Z6, Z6
	VXORPD         Z5, Z6, Z5

	// Store 1 outputs
	VMOVDQU64 Z5, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x1_64Xor_loop
	VZEROUPPER

mulGFNI_5x1_64Xor_end:
	RET

// func mulAvxGFNI_5x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 8 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), CX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R8
	MOVQ         start+72(FP), R9

	// Add start offset to output
	ADDQ R9, R8

	// Add start offset to input
	ADDQ R9, DX
	ADDQ R9, BX
	ADDQ R9, SI
	ADDQ R9, DI
	ADDQ R9, CX

mulAvxGFNI_5x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (R8), Y5

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y6
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y6
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y6
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y6
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (CX), Y6
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y4, Y6, Y6
	VXORPD         Y5, Y6, Y5

	// Store 1 outputs
	VMOVDQU Y5, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x1Xor_loop
	VZEROUPPER

mulAvxGFNI_5x1Xor_end:
	RET

// func mulAvxTwo_5x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R9
	MOVQ  start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, R8
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R10
	MOVQ         R10, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_5x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (R9), Y0
	VMOVDQU 32(R9), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R9)
	VMOVDQU Y1, 32(R9)
	ADDQ    $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_5x1_64Xor_end:
	RET

// func mulAvxTwo_5x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 49 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R9
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10
	ADDQ R11, R9

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_5x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R10)
	VMOVDQU Y1, 32(R10)
	ADDQ    $0x40, R10
	VMOVDQU Y2, (R9)
	VMOVDQU Y3, 32(R9)
	ADDQ    $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x2_64_loop
	VZEROUPPER

mulAvxTwo_5x2_64_end:
	RET

// func mulGFNI_5x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R8
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9
	ADDQ R10, R8

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, CX

mulGFNI_5x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z12
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z12, Z10
	VGF2P8AFFINEQB $0x00, Z1, Z12, Z11

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z12
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z12
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z12
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (CX), Z12
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z9, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Store 2 outputs
	VMOVDQU64 Z10, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z11, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x2_64_loop
	VZEROUPPER

mulGFNI_5x2_64_end:
	RET

// func mulAvxGFNI_5x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x2(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), CX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R8
	MOVQ         start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9
	ADDQ R10, R8

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, CX

mulAvxGFNI_5x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y12
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y12, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y12, Y11

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y12
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (SI), Y12
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (DI), Y12
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (CX), Y12
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Store 2 outputs
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x2_loop
	VZEROUPPER

mulAvxGFNI_5x2_end:
	RET

// func mulGFNI_5x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R8
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9
	ADDQ R10, R8

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, CX

mulGFNI_5x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (R9), Z10
	VMOVDQU64 (R8), Z11

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z12
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z12
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z12
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z12
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (CX), Z12
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
	VXORPD         Z10, Z13, Z10
	VGF2P8AFFINEQB $0x00, Z9, Z12, Z13
	VXORPD         Z11, Z13, Z11

	// Store 2 outputs
	VMOVDQU64 Z10, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z11, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x2_64Xor_loop
	VZEROUPPER

mulGFNI_5x2_64Xor_end:
	RET

// func mulAvxGFNI_5x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x2Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 14 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), CX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R8
	MOVQ         start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9
	ADDQ R10, R8

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, CX

mulAvxGFNI_5x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (R9), Y10
	VMOVDQU (R8), Y11

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y12
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y12
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (SI), Y12
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (DI), Y12
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (CX), Y12
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
	VXORPD         Y10, Y13, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y12, Y13
	VXORPD         Y11, Y13, Y11

	// Store 2 outputs
	VMOVDQU Y10, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y11, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x2Xor_loop
	VZEROUPPER

mulAvxGFNI_5x2Xor_end:
	RET

// func mulAvxTwo_5x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 49 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R9
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10
	ADDQ R11, R9

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_5x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R10), Y0
	VMOVDQU 32(R10), Y1
	VMOVDQU (R9), Y2
	VMOVDQU 32(R9), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R10)
	VMOVDQU Y1, 32(R10)
	ADDQ    $0x40, R10
	VMOVDQU Y2, (R9)
	VMOVDQU Y3, 32(R9)
	ADDQ    $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_5x2_64Xor_end:
	RET

// func mulAvxTwo_5x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 70 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R9
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R9

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_5x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R10)
	VMOVDQU Y1, 32(R10)
	ADDQ    $0x40, R10
	VMOVDQU Y2, (R11)
	VMOVDQU Y3, 32(R11)
	ADDQ    $0x40, R11
	VMOVDQU Y4, (R9)
	VMOVDQU Y5, 32(R9)
	ADDQ    $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x3_64_loop
	VZEROUPPER

mulAvxTwo_5x3_64_end:
	RET

// func mulGFNI_5x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R8
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, R8

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, CX

mulGFNI_5x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z18
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z18, Z15
	VGF2P8AFFINEQB $0x00, Z1, Z18, Z16
	VGF2P8AFFINEQB $0x00, Z2, Z18, Z17

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z18
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z18
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z18
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (CX), Z18
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Store 3 outputs
	VMOVDQU64 Z15, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z16, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z17, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x3_64_loop
	VZEROUPPER

mulGFNI_5x3_64_end:
	RET

// func mulAvxGFNI_5x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x3(SB), $0-88
	// Loading 11 of 15 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R9
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R9

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, DX

mulAvxGFNI_5x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x3_loop
	VZEROUPPER

mulAvxGFNI_5x3_end:
	RET

// func mulGFNI_5x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R8
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R9
	ADDQ R11, R10
	ADDQ R11, R8

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, CX

mulGFNI_5x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (R9), Z15
	VMOVDQU64 (R10), Z16
	VMOVDQU64 (R8), Z17

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z18
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z18
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z18
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z18
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (CX), Z18
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
	VXORPD         Z15, Z19, Z15
	VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Store 3 outputs
	VMOVDQU64 Z15, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z16, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z17, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x3_64Xor_loop
	VZEROUPPER

mulGFNI_5x3_64Xor_end:
	RET

// func mulAvxGFNI_5x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x3Xor(SB), $0-88
	// Loading 11 of 15 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R9
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R9

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, DX

mulAvxGFNI_5x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU (R11), Y12
	VMOVDQU (R9), Y13

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y12, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x3Xor_loop
	VZEROUPPER

mulAvxGFNI_5x3Xor_end:
	RET

// func mulAvxTwo_5x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 70 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R9
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R9

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_5x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R10), Y0
	VMOVDQU 32(R10), Y1
	VMOVDQU (R11), Y2
	VMOVDQU 32(R11), Y3
	VMOVDQU (R9), Y4
	VMOVDQU 32(R9), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R10)
	VMOVDQU Y1, 32(R10)
	ADDQ    $0x40, R10
	VMOVDQU Y2, (R11)
	VMOVDQU Y3, 32(R11)
	ADDQ    $0x40, R11
	VMOVDQU Y4, (R9)
	VMOVDQU Y5, 32(R9)
	ADDQ    $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_5x3_64Xor_end:
	RET

// func mulAvxTwo_5x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x4(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 49 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R9
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R9

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_5x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x4_loop
	VZEROUPPER

mulAvxTwo_5x4_end:
	RET

// func mulGFNI_5x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x4_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R8
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R8

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, CX

mulGFNI_5x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z23

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z24
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z24
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (DI), Z24
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 4 outputs
	VMOVDQU64 Z20, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z21, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z22, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z23, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x4_64_loop
	VZEROUPPER

mulGFNI_5x4_64_end:
	RET

// func mulAvxGFNI_5x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x4(SB), $0-88
	// Loading 10 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R9
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R9

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, DX

mulAvxGFNI_5x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x4_loop
	VZEROUPPER

mulAvxGFNI_5x4_end:
	RET

// func mulGFNI_5x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x4_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R8
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R8

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, CX

mulGFNI_5x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (R9), Z20
	VMOVDQU64 (R10), Z21
	VMOVDQU64 (R11), Z22
	VMOVDQU64 (R8), Z23

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z24
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z24
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (DI), Z24
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
	VXORPD         Z20, Z25, Z20
	VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 4 outputs
	VMOVDQU64 Z20, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z21, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z22, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z23, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x4_64Xor_loop
	VZEROUPPER

mulGFNI_5x4_64Xor_end:
	RET

// func mulAvxGFNI_5x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x4Xor(SB), $0-88
	// Loading 10 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R9
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R9

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, DX

mulAvxGFNI_5x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (R10), Y10
	VMOVDQU (R11), Y11
	VMOVDQU (R12), Y12
	VMOVDQU (R9), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x4Xor_loop
	VZEROUPPER

mulAvxGFNI_5x4Xor_end:
	RET

// func mulAvxTwo_5x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x4Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 49 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R9
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R9

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_5x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (R10), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R11), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R12), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (R9), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x4Xor_loop
	VZEROUPPER

mulAvxTwo_5x4Xor_end:
	RET

// func mulAvxTwo_5x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x5(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 60 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R9
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R9

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_5x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x5_loop
	VZEROUPPER

mulAvxTwo_5x5_end:
	RET

// func mulGFNI_5x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x5_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R8
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R8

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, CX

mulGFNI_5x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z26, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z27, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z28, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x5_64_loop
	VZEROUPPER

mulGFNI_5x5_64_end:
	RET

// func mulAvxGFNI_5x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x5(SB), $0-88
	// Loading 9 of 25 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R9
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R9

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, DX

mulAvxGFNI_5x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x5_loop
	VZEROUPPER

mulAvxGFNI_5x5_end:
	RET

// func mulGFNI_5x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x5_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), CX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R8
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R8

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, CX

mulGFNI_5x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (R9), Z25
	VMOVDQU64 (R10), Z26
	VMOVDQU64 (R11), Z27
	VMOVDQU64 (R12), Z28
	VMOVDQU64 (R8), Z29

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z26, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z27, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z28, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x5_64Xor_loop
	VZEROUPPER

mulGFNI_5x5_64Xor_end:
	RET

// func mulAvxGFNI_5x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x5Xor(SB), $0-88
	// Loading 9 of 25 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R9
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R9

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, DX

mulAvxGFNI_5x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (R10), Y9
	VMOVDQU (R11), Y10
	VMOVDQU (R12), Y11
	VMOVDQU (R13), Y12
	VMOVDQU (R9), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x5Xor_loop
	VZEROUPPER

mulAvxGFNI_5x5Xor_end:
	RET

// func mulAvxTwo_5x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x5Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 60 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R9
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R9

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_5x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (R10), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (R11), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R12), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R13), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (R9), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x5Xor_loop
	VZEROUPPER

mulAvxTwo_5x5Xor_end:
	RET

// func mulAvxTwo_5x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x6(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 71 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R14
	MOVQ  120(R9), R9
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R9

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_5x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y5, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x6_loop
	VZEROUPPER

mulAvxTwo_5x6_end:
	RET

// func mulGFNI_5x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x6_64(SB), $0-88
	// Loading 24 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R13
	MOVQ            96(R9), R14
	MOVQ            120(R9), R9
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R9

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, DX

mulGFNI_5x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x6_64_loop
	VZEROUPPER

mulGFNI_5x6_64_end:
	RET

// func mulAvxGFNI_5x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x6(SB), $0-88
	// Loading 8 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R14
	MOVQ         120(R9), R9
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R9

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, DX

mulAvxGFNI_5x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x6_loop
	VZEROUPPER

mulAvxGFNI_5x6_end:
	RET

// func mulGFNI_5x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x6_64Xor(SB), $0-88
	// Loading 24 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R13
	MOVQ            96(R9), R14
	MOVQ            120(R9), R9
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R9

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, DX

mulGFNI_5x6_64Xor_loop:
	// Load 6 outputs
	VMOVDQU64 (R10), Z24
	VMOVDQU64 (R11), Z25
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (R13), Z27
	VMOVDQU64 (R14), Z28
	VMOVDQU64 (R9), Z29

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x6_64Xor_loop
	VZEROUPPER

mulGFNI_5x6_64Xor_end:
	RET

// func mulAvxGFNI_5x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x6Xor(SB), $0-88
	// Loading 8 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R14
	MOVQ         120(R9), R9
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R9

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, DX

mulAvxGFNI_5x6Xor_loop:
	// Load 6 outputs
	VMOVDQU (R10), Y8
	VMOVDQU (R11), Y9
	VMOVDQU (R12), Y10
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (R9), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x6Xor_loop
	VZEROUPPER

mulAvxGFNI_5x6Xor_end:
	RET

// func mulAvxTwo_5x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x6Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 71 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R14
	MOVQ  120(R9), R9
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R9

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_5x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (R10), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU (R11), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU (R12), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU (R13), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU (R14), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU (R9), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y5, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x6Xor_loop
	VZEROUPPER

mulAvxTwo_5x6Xor_end:
	RET

// func mulAvxTwo_5x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x7(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 82 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R14
	MOVQ  120(R9), R15
	MOVQ  144(R9), R9
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_5x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y5, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x7_loop
	VZEROUPPER

mulAvxTwo_5x7_end:
	RET

// func mulGFNI_5x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x7_64(SB), $8-88
	// Loading 23 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R13
	MOVQ            96(R9), R14
	MOVQ            120(R9), R15
	MOVQ            144(R9), R9
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, DX

mulGFNI_5x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x7_64_loop
	VZEROUPPER

mulGFNI_5x7_64_end:
	RET

// func mulAvxGFNI_5x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x7(SB), $8-88
	// Loading 7 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R14
	MOVQ         120(R9), R15
	MOVQ         144(R9), R9
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, DX

mulAvxGFNI_5x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x7_loop
	VZEROUPPER

mulAvxGFNI_5x7_end:
	RET

// func mulGFNI_5x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x7_64Xor(SB), $8-88
	// Loading 23 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R13
	MOVQ            96(R9), R14
	MOVQ            120(R9), R15
	MOVQ            144(R9), R9
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, DX

mulGFNI_5x7_64Xor_loop:
	// Load 7 outputs
	VMOVDQU64 (R10), Z23
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R9), Z29

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_5x7_64Xor_loop
	VZEROUPPER

mulGFNI_5x7_64Xor_end:
	RET

// func mulAvxGFNI_5x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x7Xor(SB), $8-88
	// Loading 7 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R14
	MOVQ         120(R9), R15
	MOVQ         144(R9), R9
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, DX

mulAvxGFNI_5x7Xor_loop:
	// Load 7 outputs
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R9), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_5x7Xor_loop
	VZEROUPPER

mulAvxGFNI_5x7Xor_end:
	RET

// func mulAvxTwo_5x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x7Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 82 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R14
	MOVQ  120(R9), R15
	MOVQ  144(R9), R9
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_5x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (R10), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU (R11), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU (R12), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU (R13), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU (R14), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU (R15), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU (R9), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y5, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_5x7Xor_loop
	VZEROUPPER

mulAvxTwo_5x7Xor_end:
	RET

// func mulAvxTwo_5x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x8(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 93 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x8_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), AX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R14
	MOVQ  144(R8), R15
	MOVQ  168(R8), R8
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X8
	VPBROADCASTB X8, Y8
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_5x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (AX), Y11
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y6, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_5x8_loop
	VZEROUPPER

mulAvxTwo_5x8_end:
	RET

// func mulGFNI_5x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x8_64(SB), $8-88
	// Loading 22 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), AX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R13
	MOVQ            120(R8), R14
	MOVQ            144(R8), R15
	MOVQ            168(R8), R8
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_5x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (SI), Z30
	ADDQ                $0x40, SI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_5x8_64_loop
	VZEROUPPER

mulGFNI_5x8_64_end:
	RET

// func mulAvxGFNI_5x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x8(SB), $8-88
	// Loading 6 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), AX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R14
	MOVQ         144(R8), R15
	MOVQ         168(R8), R8
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_5x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_5x8_loop
	VZEROUPPER

mulAvxGFNI_5x8_end:
	RET

// func mulGFNI_5x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x8_64Xor(SB), $8-88
	// Loading 22 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), AX
	MOVQ            out_base+48(FP), R8
	MOVQ            out_base+48(FP), R8
	MOVQ            (R8), R9
	MOVQ            24(R8), R10
	MOVQ            48(R8), R11
	MOVQ            72(R8), R12
	MOVQ            96(R8), R13
	MOVQ            120(R8), R14
	MOVQ            144(R8), R15
	MOVQ            168(R8), R8
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_5x8_64Xor_loop:
	// Load 8 outputs
	VMOVDQU64 (R9), Z22
	VMOVDQU64 (R10), Z23
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R8), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (SI), Z30
	ADDQ                $0x40, SI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	VMOVDQU64 Z22, (R9)
	ADDQ      $0x40, R9
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R8)
	ADDQ      $0x40, R8

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_5x8_64Xor_loop
	VZEROUPPER

mulGFNI_5x8_64Xor_end:
	RET

// func mulAvxGFNI_5x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x8Xor(SB), $8-88
	// Loading 6 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), AX
	MOVQ         out_base+48(FP), R8
	MOVQ         out_base+48(FP), R8
	MOVQ         (R8), R9
	MOVQ         24(R8), R10
	MOVQ         48(R8), R11
	MOVQ         72(R8), R12
	MOVQ         96(R8), R13
	MOVQ         120(R8), R14
	MOVQ         144(R8), R15
	MOVQ         168(R8), R8
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_5x8Xor_loop:
	// Load 8 outputs
	VMOVDQU (R9), Y6
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R8), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_5x8Xor_loop
	VZEROUPPER

mulAvxGFNI_5x8Xor_end:
	RET

// func mulAvxTwo_5x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x8Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 93 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x8Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), AX
	MOVQ  out_base+48(FP), R8
	MOVQ  (R8), R9
	MOVQ  24(R8), R10
	MOVQ  48(R8), R11
	MOVQ  72(R8), R12
	MOVQ  96(R8), R13
	MOVQ  120(R8), R14
	MOVQ  144(R8), R15
	MOVQ  168(R8), R8
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R8

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X8
	VPBROADCASTB X8, Y8
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_5x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (R9), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU (R10), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU (R11), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU (R12), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU (R13), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU (R14), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU (R15), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU (R8), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (AX), Y11
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	VMOVDQU Y0, (R9)
	ADDQ    $0x20, R9
	VMOVDQU Y1, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y2, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y4, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y5, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y6, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y7, (R8)
	ADDQ    $0x20, R8

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_5x8Xor_loop
	VZEROUPPER

mulAvxTwo_5x8Xor_end:
	RET

// func mulAvxTwo_5x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x9(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 104 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  start+72(FP), R10

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, R8
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_5x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R9), R11
	VMOVDQU Y0, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y1, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y2, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y3, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y4, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y8, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxTwo_5x9_loop
	VZEROUPPER

mulAvxTwo_5x9_end:
	RET

// func mulGFNI_5x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x9_64(SB), $0-88
	// Loading 21 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulGFNI_5x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R9), R11
	VMOVDQU64 Z21, (R11)(R10*1)
	MOVQ      24(R9), R11
	VMOVDQU64 Z22, (R11)(R10*1)
	MOVQ      48(R9), R11
	VMOVDQU64 Z23, (R11)(R10*1)
	MOVQ      72(R9), R11
	VMOVDQU64 Z24, (R11)(R10*1)
	MOVQ      96(R9), R11
	VMOVDQU64 Z25, (R11)(R10*1)
	MOVQ      120(R9), R11
	VMOVDQU64 Z26, (R11)(R10*1)
	MOVQ      144(R9), R11
	VMOVDQU64 Z27, (R11)(R10*1)
	MOVQ      168(R9), R11
	VMOVDQU64 Z28, (R11)(R10*1)
	MOVQ      192(R9), R11
	VMOVDQU64 Z29, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x40, R10
	DECQ AX
	JNZ  mulGFNI_5x9_64_loop
	VZEROUPPER

mulGFNI_5x9_64_end:
	RET

// func mulAvxGFNI_5x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x9(SB), $0-88
	// Loading 5 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulAvxGFNI_5x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y8, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y9, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y10, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y11, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y12, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y13, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxGFNI_5x9_loop
	VZEROUPPER

mulAvxGFNI_5x9_end:
	RET

// func mulGFNI_5x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x9_64Xor(SB), $0-88
	// Loading 21 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulGFNI_5x9_64Xor_loop:
	// Load 9 outputs
	MOVQ      (R9), R11
	VMOVDQU64 (R11)(R10*1), Z21
	MOVQ      24(R9), R11
	VMOVDQU64 (R11)(R10*1), Z22
	MOVQ      48(R9), R11
	VMOVDQU64 (R11)(R10*1), Z23
	MOVQ      72(R9), R11
	VMOVDQU64 (R11)(R10*1), Z24
	MOVQ      96(R9), R11
	VMOVDQU64 (R11)(R10*1), Z25
	MOVQ      120(R9), R11
	VMOVDQU64 (R11)(R10*1), Z26
	MOVQ      144(R9), R11
	VMOVDQU64 (R11)(R10*1), Z27
	MOVQ      168(R9), R11
	VMOVDQU64 (R11)(R10*1), Z28
	MOVQ      192(R9), R11
	VMOVDQU64 (R11)(R10*1), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R9), R11
	VMOVDQU64 Z21, (R11)(R10*1)
	MOVQ      24(R9), R11
	VMOVDQU64 Z22, (R11)(R10*1)
	MOVQ      48(R9), R11
	VMOVDQU64 Z23, (R11)(R10*1)
	MOVQ      72(R9), R11
	VMOVDQU64 Z24, (R11)(R10*1)
	MOVQ      96(R9), R11
	VMOVDQU64 Z25, (R11)(R10*1)
	MOVQ      120(R9), R11
	VMOVDQU64 Z26, (R11)(R10*1)
	MOVQ      144(R9), R11
	VMOVDQU64 Z27, (R11)(R10*1)
	MOVQ      168(R9), R11
	VMOVDQU64 Z28, (R11)(R10*1)
	MOVQ      192(R9), R11
	VMOVDQU64 Z29, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x40, R10
	DECQ AX
	JNZ  mulGFNI_5x9_64Xor_loop
	VZEROUPPER

mulGFNI_5x9_64Xor_end:
	RET

// func mulAvxGFNI_5x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x9Xor(SB), $0-88
	// Loading 5 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulAvxGFNI_5x9Xor_loop:
	// Load 9 outputs
	MOVQ    (R9), R11
	VMOVDQU (R11)(R10*1), Y5
	MOVQ    24(R9), R11
	VMOVDQU (R11)(R10*1), Y6
	MOVQ    48(R9), R11
	VMOVDQU (R11)(R10*1), Y7
	MOVQ    72(R9), R11
	VMOVDQU (R11)(R10*1), Y8
	MOVQ    96(R9), R11
	VMOVDQU (R11)(R10*1), Y9
	MOVQ    120(R9), R11
	VMOVDQU (R11)(R10*1), Y10
	MOVQ    144(R9), R11
	VMOVDQU (R11)(R10*1), Y11
	MOVQ    168(R9), R11
	VMOVDQU (R11)(R10*1), Y12
	MOVQ    192(R9), R11
	VMOVDQU (R11)(R10*1), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y8, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y9, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y10, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y11, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y12, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y13, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxGFNI_5x9Xor_loop
	VZEROUPPER

mulAvxGFNI_5x9Xor_end:
	RET

// func mulAvxTwo_5x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x9Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 104 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  start+72(FP), R10

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, R8
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_5x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	MOVQ    (R9), R11
	VMOVDQU (R11)(R10*1), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	MOVQ    24(R9), R11
	VMOVDQU (R11)(R10*1), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	MOVQ    48(R9), R11
	VMOVDQU (R11)(R10*1), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	MOVQ    72(R9), R11
	VMOVDQU (R11)(R10*1), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	MOVQ    96(R9), R11
	VMOVDQU (R11)(R10*1), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	MOVQ    120(R9), R11
	VMOVDQU (R11)(R10*1), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	MOVQ    144(R9), R11
	VMOVDQU (R11)(R10*1), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	MOVQ    168(R9), R11
	VMOVDQU (R11)(R10*1), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	MOVQ    192(R9), R11
	VMOVDQU (R11)(R10*1), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R9), R11
	VMOVDQU Y0, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y1, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y2, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y3, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y4, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y8, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxTwo_5x9Xor_loop
	VZEROUPPER

mulAvxTwo_5x9Xor_end:
	RET

// func mulAvxTwo_5x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x10(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 115 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  start+72(FP), R10

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, R8
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_5x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R9), R11
	VMOVDQU Y0, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y1, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y2, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y3, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y4, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y8, (R11)(R10*1)
	MOVQ    216(R9), R11
	VMOVDQU Y9, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxTwo_5x10_loop
	VZEROUPPER

mulAvxTwo_5x10_end:
	RET

// func mulGFNI_5x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x10_64(SB), $0-88
	// Loading 20 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulGFNI_5x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R9), R11
	VMOVDQU64 Z20, (R11)(R10*1)
	MOVQ      24(R9), R11
	VMOVDQU64 Z21, (R11)(R10*1)
	MOVQ      48(R9), R11
	VMOVDQU64 Z22, (R11)(R10*1)
	MOVQ      72(R9), R11
	VMOVDQU64 Z23, (R11)(R10*1)
	MOVQ      96(R9), R11
	VMOVDQU64 Z24, (R11)(R10*1)
	MOVQ      120(R9), R11
	VMOVDQU64 Z25, (R11)(R10*1)
	MOVQ      144(R9), R11
	VMOVDQU64 Z26, (R11)(R10*1)
	MOVQ      168(R9), R11
	VMOVDQU64 Z27, (R11)(R10*1)
	MOVQ      192(R9), R11
	VMOVDQU64 Z28, (R11)(R10*1)
	MOVQ      216(R9), R11
	VMOVDQU64 Z29, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x40, R10
	DECQ AX
	JNZ  mulGFNI_5x10_64_loop
	VZEROUPPER

mulGFNI_5x10_64_end:
	RET

// func mulAvxGFNI_5x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x10(SB), $0-88
	// Loading 4 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulAvxGFNI_5x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R9), R11
	VMOVDQU Y4, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y8, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y9, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y10, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y11, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y12, (R11)(R10*1)
	MOVQ    216(R9), R11
	VMOVDQU Y13, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxGFNI_5x10_loop
	VZEROUPPER

mulAvxGFNI_5x10_end:
	RET

// func mulGFNI_5x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_5x10_64Xor(SB), $0-88
	// Loading 20 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_5x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), DX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulGFNI_5x10_64Xor_loop:
	// Load 10 outputs
	MOVQ      (R9), R11
	VMOVDQU64 (R11)(R10*1), Z20
	MOVQ      24(R9), R11
	VMOVDQU64 (R11)(R10*1), Z21
	MOVQ      48(R9), R11
	VMOVDQU64 (R11)(R10*1), Z22
	MOVQ      72(R9), R11
	VMOVDQU64 (R11)(R10*1), Z23
	MOVQ      96(R9), R11
	VMOVDQU64 (R11)(R10*1), Z24
	MOVQ      120(R9), R11
	VMOVDQU64 (R11)(R10*1), Z25
	MOVQ      144(R9), R11
	VMOVDQU64 (R11)(R10*1), Z26
	MOVQ      168(R9), R11
	VMOVDQU64 (R11)(R10*1), Z27
	MOVQ      192(R9), R11
	VMOVDQU64 (R11)(R10*1), Z28
	MOVQ      216(R9), R11
	VMOVDQU64 (R11)(R10*1), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R9), R11
	VMOVDQU64 Z20, (R11)(R10*1)
	MOVQ      24(R9), R11
	VMOVDQU64 Z21, (R11)(R10*1)
	MOVQ      48(R9), R11
	VMOVDQU64 Z22, (R11)(R10*1)
	MOVQ      72(R9), R11
	VMOVDQU64 Z23, (R11)(R10*1)
	MOVQ      96(R9), R11
	VMOVDQU64 Z24, (R11)(R10*1)
	MOVQ      120(R9), R11
	VMOVDQU64 Z25, (R11)(R10*1)
	MOVQ      144(R9), R11
	VMOVDQU64 Z26, (R11)(R10*1)
	MOVQ      168(R9), R11
	VMOVDQU64 Z27, (R11)(R10*1)
	MOVQ      192(R9), R11
	VMOVDQU64 Z28, (R11)(R10*1)
	MOVQ      216(R9), R11
	VMOVDQU64 Z29, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x40, R10
	DECQ AX
	JNZ  mulGFNI_5x10_64Xor_loop
	VZEROUPPER

mulGFNI_5x10_64Xor_end:
	RET

// func mulAvxGFNI_5x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_5x10Xor(SB), $0-88
	// Loading 4 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_5x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), DX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         start+72(FP), R10

	// Add start offset to input
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, DX

mulAvxGFNI_5x10Xor_loop:
	// Load 10 outputs
	MOVQ    (R9), R11
	VMOVDQU (R11)(R10*1), Y4
	MOVQ    24(R9), R11
	VMOVDQU (R11)(R10*1), Y5
	MOVQ    48(R9), R11
	VMOVDQU (R11)(R10*1), Y6
	MOVQ    72(R9), R11
	VMOVDQU (R11)(R10*1), Y7
	MOVQ    96(R9), R11
	VMOVDQU (R11)(R10*1), Y8
	MOVQ    120(R9), R11
	VMOVDQU (R11)(R10*1), Y9
	MOVQ    144(R9), R11
	VMOVDQU (R11)(R10*1), Y10
	MOVQ    168(R9), R11
	VMOVDQU (R11)(R10*1), Y11
	MOVQ    192(R9), R11
	VMOVDQU (R11)(R10*1), Y12
	MOVQ    216(R9), R11
	VMOVDQU (R11)(R10*1), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R9), R11
	VMOVDQU Y4, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y8, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y9, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y10, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y11, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y12, (R11)(R10*1)
	MOVQ    216(R9), R11
	VMOVDQU Y13, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxGFNI_5x10Xor_loop
	VZEROUPPER

mulAvxGFNI_5x10Xor_end:
	RET

// func mulAvxTwo_5x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_5x10Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 115 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_5x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), DX
	MOVQ  out_base+48(FP), R9
	MOVQ  start+72(FP), R10

	// Add start offset to input
	ADDQ         R10, BX
	ADDQ         R10, SI
	ADDQ         R10, DI
	ADDQ         R10, R8
	ADDQ         R10, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_5x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	MOVQ    (R9), R11
	VMOVDQU (R11)(R10*1), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	MOVQ    24(R9), R11
	VMOVDQU (R11)(R10*1), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	MOVQ    48(R9), R11
	VMOVDQU (R11)(R10*1), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	MOVQ    72(R9), R11
	VMOVDQU (R11)(R10*1), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	MOVQ    96(R9), R11
	VMOVDQU (R11)(R10*1), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	MOVQ    120(R9), R11
	VMOVDQU (R11)(R10*1), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	MOVQ    144(R9), R11
	VMOVDQU (R11)(R10*1), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	MOVQ    168(R9), R11
	VMOVDQU (R11)(R10*1), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	MOVQ    192(R9), R11
	VMOVDQU (R11)(R10*1), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	MOVQ    216(R9), R11
	VMOVDQU (R11)(R10*1), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R9), R11
	VMOVDQU Y0, (R11)(R10*1)
	MOVQ    24(R9), R11
	VMOVDQU Y1, (R11)(R10*1)
	MOVQ    48(R9), R11
	VMOVDQU Y2, (R11)(R10*1)
	MOVQ    72(R9), R11
	VMOVDQU Y3, (R11)(R10*1)
	MOVQ    96(R9), R11
	VMOVDQU Y4, (R11)(R10*1)
	MOVQ    120(R9), R11
	VMOVDQU Y5, (R11)(R10*1)
	MOVQ    144(R9), R11
	VMOVDQU Y6, (R11)(R10*1)
	MOVQ    168(R9), R11
	VMOVDQU Y7, (R11)(R10*1)
	MOVQ    192(R9), R11
	VMOVDQU Y8, (R11)(R10*1)
	MOVQ    216(R9), R11
	VMOVDQU Y9, (R11)(R10*1)

	// Prepare for next loop
	ADDQ $0x20, R10
	DECQ AX
	JNZ  mulAvxTwo_5x10Xor_loop
	VZEROUPPER

mulAvxTwo_5x10Xor_end:
	RET

// func mulAvxTwo_6x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R10
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_6x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R10)
	VMOVDQU Y1, 32(R10)
	ADDQ    $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x1_64_loop
	VZEROUPPER

mulAvxTwo_6x1_64_end:
	RET

// func mulGFNI_6x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 9 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R9
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, CX

mulGFNI_6x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z7
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z7, Z6

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z7
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z7
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z7
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z7
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (CX), Z7
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z5, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Store 1 outputs
	VMOVDQU64 Z6, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x1_64_loop
	VZEROUPPER

mulGFNI_6x1_64_end:
	RET

// func mulAvxGFNI_6x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 9 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), CX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R9
	MOVQ         start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, CX

mulAvxGFNI_6x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y7
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y7, Y6

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y7
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y7
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y7
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y7
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (CX), Y7
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y5, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Store 1 outputs
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x1_loop
	VZEROUPPER

mulAvxGFNI_6x1_end:
	RET

// func mulGFNI_6x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 9 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R9
	MOVQ            start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, CX

mulGFNI_6x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (R9), Z6

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z7
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z7
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z7
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z7
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z7
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (CX), Z7
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z5, Z7, Z7
	VXORPD         Z6, Z7, Z6

	// Store 1 outputs
	VMOVDQU64 Z6, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x1_64Xor_loop
	VZEROUPPER

mulGFNI_6x1_64Xor_end:
	RET

// func mulAvxGFNI_6x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 9 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), CX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R9
	MOVQ         start+72(FP), R10

	// Add start offset to output
	ADDQ R10, R9

	// Add start offset to input
	ADDQ R10, DX
	ADDQ R10, BX
	ADDQ R10, SI
	ADDQ R10, DI
	ADDQ R10, R8
	ADDQ R10, CX

mulAvxGFNI_6x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (R9), Y6

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y7
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y7
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y7
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y7
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y7
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (CX), Y7
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y5, Y7, Y7
	VXORPD         Y6, Y7, Y6

	// Store 1 outputs
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x1Xor_loop
	VZEROUPPER

mulAvxGFNI_6x1Xor_end:
	RET

// func mulAvxTwo_6x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R10
	MOVQ  start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R11
	MOVQ         R11, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_6x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (R10), Y0
	VMOVDQU 32(R10), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R10)
	VMOVDQU Y1, 32(R10)
	ADDQ    $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_6x1_64Xor_end:
	RET

// func mulAvxTwo_6x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 57 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R10
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11
	ADDQ R12, R10

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_6x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R11)
	VMOVDQU Y1, 32(R11)
	ADDQ    $0x40, R11
	VMOVDQU Y2, (R10)
	VMOVDQU Y3, 32(R10)
	ADDQ    $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x2_64_loop
	VZEROUPPER

mulAvxTwo_6x2_64_end:
	RET

// func mulGFNI_6x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R9
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10
	ADDQ R11, R9

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, CX

mulGFNI_6x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z14
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z14, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z14, Z13

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z14
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z3, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z14
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z5, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z14
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z7, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z14
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z9, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (CX), Z14
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z10, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z11, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Store 2 outputs
	VMOVDQU64 Z12, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x2_64_loop
	VZEROUPPER

mulGFNI_6x2_64_end:
	RET

// func mulAvxGFNI_6x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x2(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), CX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R9
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10
	ADDQ R11, R9

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, CX

mulAvxGFNI_6x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (CX), Y14
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x2_loop
	VZEROUPPER

mulAvxGFNI_6x2_end:
	RET

// func mulGFNI_6x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R9
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10
	ADDQ R11, R9

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, CX

mulGFNI_6x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (R10), Z12
	VMOVDQU64 (R9), Z13

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z14
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z1, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z14
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z3, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z14
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z5, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z14
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z7, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z14
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z9, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (CX), Z14
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z10, Z14, Z15
	VXORPD         Z12, Z15, Z12
	VGF2P8AFFINEQB $0x00, Z11, Z14, Z15
	VXORPD         Z13, Z15, Z13

	// Store 2 outputs
	VMOVDQU64 Z12, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z13, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x2_64Xor_loop
	VZEROUPPER

mulGFNI_6x2_64Xor_end:
	RET

// func mulAvxGFNI_6x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x2Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 16 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), CX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R9
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10
	ADDQ R11, R9

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, CX

mulAvxGFNI_6x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (R10), Y12
	VMOVDQU (R9), Y13

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (CX), Y14
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x2Xor_loop
	VZEROUPPER

mulAvxGFNI_6x2Xor_end:
	RET

// func mulAvxTwo_6x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 57 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R10
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11
	ADDQ R12, R10

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_6x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R11), Y0
	VMOVDQU 32(R11), Y1
	VMOVDQU (R10), Y2
	VMOVDQU 32(R10), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R11)
	VMOVDQU Y1, 32(R11)
	ADDQ    $0x40, R11
	VMOVDQU Y2, (R10)
	VMOVDQU Y3, 32(R10)
	ADDQ    $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_6x2_64Xor_end:
	RET

// func mulAvxTwo_6x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 82 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R10
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R10

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_6x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R11)
	VMOVDQU Y1, 32(R11)
	ADDQ    $0x40, R11
	VMOVDQU Y2, (R12)
	VMOVDQU Y3, 32(R12)
	ADDQ    $0x40, R12
	VMOVDQU Y4, (R10)
	VMOVDQU Y5, 32(R10)
	ADDQ    $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x3_64_loop
	VZEROUPPER

mulAvxTwo_6x3_64_end:
	RET

// func mulGFNI_6x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R9
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R9

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, CX

mulGFNI_6x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z21
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z21, Z19
	VGF2P8AFFINEQB $0x00, Z2, Z21, Z20

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z21
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z4, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z5, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z21
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z21
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z21
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z14, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (CX), Z21
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z15, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z16, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z17, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Store 3 outputs
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z20, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x3_64_loop
	VZEROUPPER

mulGFNI_6x3_64_end:
	RET

// func mulAvxGFNI_6x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x3(SB), $0-88
	// Loading 11 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R10
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R10

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, DX

mulAvxGFNI_6x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x3_loop
	VZEROUPPER

mulAvxGFNI_6x3_end:
	RET

// func mulGFNI_6x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R9
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R10
	ADDQ R12, R11
	ADDQ R12, R9

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, CX

mulGFNI_6x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (R10), Z18
	VMOVDQU64 (R11), Z19
	VMOVDQU64 (R9), Z20

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z21
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z2, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z21
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z4, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z5, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z21
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z21
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z21
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z14, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (CX), Z21
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z15, Z21, Z22
	VXORPD         Z18, Z22, Z18
	VGF2P8AFFINEQB $0x00, Z16, Z21, Z22
	VXORPD         Z19, Z22, Z19
	VGF2P8AFFINEQB $0x00, Z17, Z21, Z22
	VXORPD         Z20, Z22, Z20

	// Store 3 outputs
	VMOVDQU64 Z18, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z19, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z20, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x3_64Xor_loop
	VZEROUPPER

mulGFNI_6x3_64Xor_end:
	RET

// func mulAvxGFNI_6x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x3Xor(SB), $0-88
	// Loading 11 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 23 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R10
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R10

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, DX

mulAvxGFNI_6x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (R11), Y11
	VMOVDQU (R12), Y12
	VMOVDQU (R10), Y13

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x3Xor_loop
	VZEROUPPER

mulAvxGFNI_6x3Xor_end:
	RET

// func mulAvxTwo_6x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 82 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R10
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R10

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_6x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R11), Y0
	VMOVDQU 32(R11), Y1
	VMOVDQU (R12), Y2
	VMOVDQU 32(R12), Y3
	VMOVDQU (R10), Y4
	VMOVDQU 32(R10), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R11)
	VMOVDQU Y1, 32(R11)
	ADDQ    $0x40, R11
	VMOVDQU Y2, (R12)
	VMOVDQU Y3, 32(R12)
	ADDQ    $0x40, R12
	VMOVDQU Y4, (R10)
	VMOVDQU Y5, 32(R10)
	ADDQ    $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_6x3_64Xor_end:
	RET

// func mulAvxTwo_6x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x4(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 57 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R10
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R10

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_6x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x4_loop
	VZEROUPPER

mulAvxTwo_6x4_end:
	RET

// func mulGFNI_6x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x4_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R9
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R9

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, CX

mulGFNI_6x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z28
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z28, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z28, Z27

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z28
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z28
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (DI), Z28
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R8), Z28
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (CX), Z28
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z21, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z22, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z23, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Store 4 outputs
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x4_64_loop
	VZEROUPPER

mulGFNI_6x4_64_end:
	RET

// func mulAvxGFNI_6x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x4(SB), $0-88
	// Loading 10 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R10
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R10

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, DX

mulAvxGFNI_6x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x4_loop
	VZEROUPPER

mulAvxGFNI_6x4_end:
	RET

// func mulGFNI_6x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x4_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), CX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R9
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R9

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, CX

mulGFNI_6x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (R10), Z24
	VMOVDQU64 (R11), Z25
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (R9), Z27

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z28
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z28
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z28
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (DI), Z28
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R8), Z28
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (CX), Z28
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
	VXORPD         Z24, Z29, Z24
	VGF2P8AFFINEQB $0x00, Z21, Z28, Z29
	VXORPD         Z25, Z29, Z25
	VGF2P8AFFINEQB $0x00, Z22, Z28, Z29
	VXORPD         Z26, Z29, Z26
	VGF2P8AFFINEQB $0x00, Z23, Z28, Z29
	VXORPD         Z27, Z29, Z27

	// Store 4 outputs
	VMOVDQU64 Z24, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x4_64Xor_loop
	VZEROUPPER

mulGFNI_6x4_64Xor_end:
	RET

// func mulAvxGFNI_6x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x4Xor(SB), $0-88
	// Loading 10 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 30 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R10
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R10

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, DX

mulAvxGFNI_6x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (R11), Y10
	VMOVDQU (R12), Y11
	VMOVDQU (R13), Y12
	VMOVDQU (R10), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x4Xor_loop
	VZEROUPPER

mulAvxGFNI_6x4Xor_end:
	RET

// func mulAvxTwo_6x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x4Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 57 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R10
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R10

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_6x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (R11), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R12), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R13), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (R10), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x4Xor_loop
	VZEROUPPER

mulAvxTwo_6x4Xor_end:
	RET

// func mulAvxTwo_6x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x5(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 70 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R14
	MOVQ  96(R10), R10
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R10

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_6x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x5_loop
	VZEROUPPER

mulAvxTwo_6x5_end:
	RET

// func mulGFNI_6x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x5_64(SB), $0-88
	// Loading 25 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R13
	MOVQ            72(R10), R14
	MOVQ            96(R10), R10
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R10

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, DX

mulGFNI_6x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x5_64_loop
	VZEROUPPER

mulGFNI_6x5_64_end:
	RET

// func mulAvxGFNI_6x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x5(SB), $0-88
	// Loading 9 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R14
	MOVQ         96(R10), R10
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R10

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, DX

mulAvxGFNI_6x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x5_loop
	VZEROUPPER

mulAvxGFNI_6x5_end:
	RET

// func mulGFNI_6x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x5_64Xor(SB), $0-88
	// Loading 25 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R13
	MOVQ            72(R10), R14
	MOVQ            96(R10), R10
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R10

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, DX

mulGFNI_6x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (R11), Z25
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (R13), Z27
	VMOVDQU64 (R14), Z28
	VMOVDQU64 (R10), Z29

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x5_64Xor_loop
	VZEROUPPER

mulGFNI_6x5_64Xor_end:
	RET

// func mulAvxGFNI_6x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x5Xor(SB), $0-88
	// Loading 9 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 37 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R14
	MOVQ         96(R10), R10
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R10

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, DX

mulAvxGFNI_6x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (R11), Y9
	VMOVDQU (R12), Y10
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (R10), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x5Xor_loop
	VZEROUPPER

mulAvxGFNI_6x5Xor_end:
	RET

// func mulAvxTwo_6x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x5Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 70 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R14
	MOVQ  96(R10), R10
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R10

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_6x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (R11), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (R12), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R13), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R14), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (R10), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y4, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x5Xor_loop
	VZEROUPPER

mulAvxTwo_6x5Xor_end:
	RET

// func mulAvxTwo_6x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x6(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 83 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R14
	MOVQ  96(R10), R15
	MOVQ  120(R10), R10
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_6x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y4, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y5, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x6_loop
	VZEROUPPER

mulAvxTwo_6x6_end:
	RET

// func mulGFNI_6x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x6_64(SB), $8-88
	// Loading 24 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R13
	MOVQ            72(R10), R14
	MOVQ            96(R10), R15
	MOVQ            120(R10), R10
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, DX

mulGFNI_6x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x6_64_loop
	VZEROUPPER

mulGFNI_6x6_64_end:
	RET

// func mulAvxGFNI_6x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x6(SB), $8-88
	// Loading 8 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R14
	MOVQ         96(R10), R15
	MOVQ         120(R10), R10
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, DX

mulAvxGFNI_6x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x6_loop
	VZEROUPPER

mulAvxGFNI_6x6_end:
	RET

// func mulGFNI_6x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x6_64Xor(SB), $8-88
	// Loading 24 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R13
	MOVQ            72(R10), R14
	MOVQ            96(R10), R15
	MOVQ            120(R10), R10
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, DX

mulGFNI_6x6_64Xor_loop:
	// Load 6 outputs
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R10), Z29

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_6x6_64Xor_loop
	VZEROUPPER

mulGFNI_6x6_64Xor_end:
	RET

// func mulAvxGFNI_6x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x6Xor(SB), $8-88
	// Loading 8 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 44 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R14
	MOVQ         96(R10), R15
	MOVQ         120(R10), R10
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, DX

mulAvxGFNI_6x6Xor_loop:
	// Load 6 outputs
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R10), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_6x6Xor_loop
	VZEROUPPER

mulAvxGFNI_6x6Xor_end:
	RET

// func mulAvxTwo_6x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x6Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 83 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R14
	MOVQ  96(R10), R15
	MOVQ  120(R10), R10
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_6x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (R11), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU (R12), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU (R13), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU (R14), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU (R15), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU (R10), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y4, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y5, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_6x6Xor_loop
	VZEROUPPER

mulAvxTwo_6x6Xor_end:
	RET

// func mulAvxTwo_6x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x7(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 96 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x7_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), AX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R14
	MOVQ  120(R9), R15
	MOVQ  144(R9), R9
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X7
	VPBROADCASTB X7, Y7
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_6x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (AX), Y10
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y5, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_6x7_loop
	VZEROUPPER

mulAvxTwo_6x7_end:
	RET

// func mulGFNI_6x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x7_64(SB), $8-88
	// Loading 23 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 51 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), AX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R13
	MOVQ            96(R9), R14
	MOVQ            120(R9), R15
	MOVQ            144(R9), R9
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_6x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_6x7_64_loop
	VZEROUPPER

mulGFNI_6x7_64_end:
	RET

// func mulAvxGFNI_6x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x7(SB), $8-88
	// Loading 7 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 51 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), AX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R14
	MOVQ         120(R9), R15
	MOVQ         144(R9), R9
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_6x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_6x7_loop
	VZEROUPPER

mulAvxGFNI_6x7_end:
	RET

// func mulGFNI_6x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x7_64Xor(SB), $8-88
	// Loading 23 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 51 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), AX
	MOVQ            out_base+48(FP), R9
	MOVQ            out_base+48(FP), R9
	MOVQ            (R9), R10
	MOVQ            24(R9), R11
	MOVQ            48(R9), R12
	MOVQ            72(R9), R13
	MOVQ            96(R9), R14
	MOVQ            120(R9), R15
	MOVQ            144(R9), R9
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_6x7_64Xor_loop:
	// Load 7 outputs
	VMOVDQU64 (R10), Z23
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R9), Z29

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R9)
	ADDQ      $0x40, R9

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_6x7_64Xor_loop
	VZEROUPPER

mulGFNI_6x7_64Xor_end:
	RET

// func mulAvxGFNI_6x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x7Xor(SB), $8-88
	// Loading 7 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 51 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), AX
	MOVQ         out_base+48(FP), R9
	MOVQ         out_base+48(FP), R9
	MOVQ         (R9), R10
	MOVQ         24(R9), R11
	MOVQ         48(R9), R12
	MOVQ         72(R9), R13
	MOVQ         96(R9), R14
	MOVQ         120(R9), R15
	MOVQ         144(R9), R9
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_6x7Xor_loop:
	// Load 7 outputs
	VMOVDQU (R10), Y7
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R9), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_6x7Xor_loop
	VZEROUPPER

mulAvxGFNI_6x7Xor_end:
	RET

// func mulAvxTwo_6x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x7Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 96 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x7Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), AX
	MOVQ  out_base+48(FP), R9
	MOVQ  (R9), R10
	MOVQ  24(R9), R11
	MOVQ  48(R9), R12
	MOVQ  72(R9), R13
	MOVQ  96(R9), R14
	MOVQ  120(R9), R15
	MOVQ  144(R9), R9
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R9

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X7
	VPBROADCASTB X7, Y7
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_6x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (R10), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU (R11), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU (R12), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU (R13), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU (R14), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU (R15), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU (R9), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (AX), Y10
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	VMOVDQU Y0, (R10)
	ADDQ    $0x20, R10
	VMOVDQU Y1, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y2, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y3, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y4, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y5, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y6, (R9)
	ADDQ    $0x20, R9

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_6x7Xor_loop
	VZEROUPPER

mulAvxTwo_6x7Xor_end:
	RET

// func mulAvxTwo_6x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x8(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 109 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  start+72(FP), R11

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_6x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R10), R12
	VMOVDQU Y0, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y1, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y2, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y3, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y7, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxTwo_6x8_loop
	VZEROUPPER

mulAvxTwo_6x8_end:
	RET

// func mulGFNI_6x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x8_64(SB), $0-88
	// Loading 22 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulGFNI_6x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R10), R12
	VMOVDQU64 Z22, (R12)(R11*1)
	MOVQ      24(R10), R12
	VMOVDQU64 Z23, (R12)(R11*1)
	MOVQ      48(R10), R12
	VMOVDQU64 Z24, (R12)(R11*1)
	MOVQ      72(R10), R12
	VMOVDQU64 Z25, (R12)(R11*1)
	MOVQ      96(R10), R12
	VMOVDQU64 Z26, (R12)(R11*1)
	MOVQ      120(R10), R12
	VMOVDQU64 Z27, (R12)(R11*1)
	MOVQ      144(R10), R12
	VMOVDQU64 Z28, (R12)(R11*1)
	MOVQ      168(R10), R12
	VMOVDQU64 Z29, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x40, R11
	DECQ AX
	JNZ  mulGFNI_6x8_64_loop
	VZEROUPPER

mulGFNI_6x8_64_end:
	RET

// func mulAvxGFNI_6x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x8(SB), $0-88
	// Loading 6 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulAvxGFNI_6x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y9, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y10, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y11, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y12, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y13, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxGFNI_6x8_loop
	VZEROUPPER

mulAvxGFNI_6x8_end:
	RET

// func mulGFNI_6x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x8_64Xor(SB), $0-88
	// Loading 22 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulGFNI_6x8_64Xor_loop:
	// Load 8 outputs
	MOVQ      (R10), R12
	VMOVDQU64 (R12)(R11*1), Z22
	MOVQ      24(R10), R12
	VMOVDQU64 (R12)(R11*1), Z23
	MOVQ      48(R10), R12
	VMOVDQU64 (R12)(R11*1), Z24
	MOVQ      72(R10), R12
	VMOVDQU64 (R12)(R11*1), Z25
	MOVQ      96(R10), R12
	VMOVDQU64 (R12)(R11*1), Z26
	MOVQ      120(R10), R12
	VMOVDQU64 (R12)(R11*1), Z27
	MOVQ      144(R10), R12
	VMOVDQU64 (R12)(R11*1), Z28
	MOVQ      168(R10), R12
	VMOVDQU64 (R12)(R11*1), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R10), R12
	VMOVDQU64 Z22, (R12)(R11*1)
	MOVQ      24(R10), R12
	VMOVDQU64 Z23, (R12)(R11*1)
	MOVQ      48(R10), R12
	VMOVDQU64 Z24, (R12)(R11*1)
	MOVQ      72(R10), R12
	VMOVDQU64 Z25, (R12)(R11*1)
	MOVQ      96(R10), R12
	VMOVDQU64 Z26, (R12)(R11*1)
	MOVQ      120(R10), R12
	VMOVDQU64 Z27, (R12)(R11*1)
	MOVQ      144(R10), R12
	VMOVDQU64 Z28, (R12)(R11*1)
	MOVQ      168(R10), R12
	VMOVDQU64 Z29, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x40, R11
	DECQ AX
	JNZ  mulGFNI_6x8_64Xor_loop
	VZEROUPPER

mulGFNI_6x8_64Xor_end:
	RET

// func mulAvxGFNI_6x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x8Xor(SB), $0-88
	// Loading 6 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulAvxGFNI_6x8Xor_loop:
	// Load 8 outputs
	MOVQ    (R10), R12
	VMOVDQU (R12)(R11*1), Y6
	MOVQ    24(R10), R12
	VMOVDQU (R12)(R11*1), Y7
	MOVQ    48(R10), R12
	VMOVDQU (R12)(R11*1), Y8
	MOVQ    72(R10), R12
	VMOVDQU (R12)(R11*1), Y9
	MOVQ    96(R10), R12
	VMOVDQU (R12)(R11*1), Y10
	MOVQ    120(R10), R12
	VMOVDQU (R12)(R11*1), Y11
	MOVQ    144(R10), R12
	VMOVDQU (R12)(R11*1), Y12
	MOVQ    168(R10), R12
	VMOVDQU (R12)(R11*1), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y9, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y10, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y11, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y12, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y13, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxGFNI_6x8Xor_loop
	VZEROUPPER

mulAvxGFNI_6x8Xor_end:
	RET

// func mulAvxTwo_6x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x8Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 109 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  start+72(FP), R11

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_6x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	MOVQ    (R10), R12
	VMOVDQU (R12)(R11*1), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	MOVQ    24(R10), R12
	VMOVDQU (R12)(R11*1), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	MOVQ    48(R10), R12
	VMOVDQU (R12)(R11*1), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	MOVQ    72(R10), R12
	VMOVDQU (R12)(R11*1), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	MOVQ    96(R10), R12
	VMOVDQU (R12)(R11*1), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	MOVQ    120(R10), R12
	VMOVDQU (R12)(R11*1), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	MOVQ    144(R10), R12
	VMOVDQU (R12)(R11*1), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	MOVQ    168(R10), R12
	VMOVDQU (R12)(R11*1), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R10), R12
	VMOVDQU Y0, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y1, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y2, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y3, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y7, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxTwo_6x8Xor_loop
	VZEROUPPER

mulAvxTwo_6x8Xor_end:
	RET

// func mulAvxTwo_6x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x9(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 122 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  start+72(FP), R11

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_6x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R10), R12
	VMOVDQU Y0, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y1, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y2, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y3, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y8, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxTwo_6x9_loop
	VZEROUPPER

mulAvxTwo_6x9_end:
	RET

// func mulGFNI_6x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x9_64(SB), $0-88
	// Loading 21 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulGFNI_6x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R10), R12
	VMOVDQU64 Z21, (R12)(R11*1)
	MOVQ      24(R10), R12
	VMOVDQU64 Z22, (R12)(R11*1)
	MOVQ      48(R10), R12
	VMOVDQU64 Z23, (R12)(R11*1)
	MOVQ      72(R10), R12
	VMOVDQU64 Z24, (R12)(R11*1)
	MOVQ      96(R10), R12
	VMOVDQU64 Z25, (R12)(R11*1)
	MOVQ      120(R10), R12
	VMOVDQU64 Z26, (R12)(R11*1)
	MOVQ      144(R10), R12
	VMOVDQU64 Z27, (R12)(R11*1)
	MOVQ      168(R10), R12
	VMOVDQU64 Z28, (R12)(R11*1)
	MOVQ      192(R10), R12
	VMOVDQU64 Z29, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x40, R11
	DECQ AX
	JNZ  mulGFNI_6x9_64_loop
	VZEROUPPER

mulGFNI_6x9_64_end:
	RET

// func mulAvxGFNI_6x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x9(SB), $0-88
	// Loading 5 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulAvxGFNI_6x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y9, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y10, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y11, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y12, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y13, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxGFNI_6x9_loop
	VZEROUPPER

mulAvxGFNI_6x9_end:
	RET

// func mulGFNI_6x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x9_64Xor(SB), $0-88
	// Loading 21 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulGFNI_6x9_64Xor_loop:
	// Load 9 outputs
	MOVQ      (R10), R12
	VMOVDQU64 (R12)(R11*1), Z21
	MOVQ      24(R10), R12
	VMOVDQU64 (R12)(R11*1), Z22
	MOVQ      48(R10), R12
	VMOVDQU64 (R12)(R11*1), Z23
	MOVQ      72(R10), R12
	VMOVDQU64 (R12)(R11*1), Z24
	MOVQ      96(R10), R12
	VMOVDQU64 (R12)(R11*1), Z25
	MOVQ      120(R10), R12
	VMOVDQU64 (R12)(R11*1), Z26
	MOVQ      144(R10), R12
	VMOVDQU64 (R12)(R11*1), Z27
	MOVQ      168(R10), R12
	VMOVDQU64 (R12)(R11*1), Z28
	MOVQ      192(R10), R12
	VMOVDQU64 (R12)(R11*1), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R10), R12
	VMOVDQU64 Z21, (R12)(R11*1)
	MOVQ      24(R10), R12
	VMOVDQU64 Z22, (R12)(R11*1)
	MOVQ      48(R10), R12
	VMOVDQU64 Z23, (R12)(R11*1)
	MOVQ      72(R10), R12
	VMOVDQU64 Z24, (R12)(R11*1)
	MOVQ      96(R10), R12
	VMOVDQU64 Z25, (R12)(R11*1)
	MOVQ      120(R10), R12
	VMOVDQU64 Z26, (R12)(R11*1)
	MOVQ      144(R10), R12
	VMOVDQU64 Z27, (R12)(R11*1)
	MOVQ      168(R10), R12
	VMOVDQU64 Z28, (R12)(R11*1)
	MOVQ      192(R10), R12
	VMOVDQU64 Z29, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x40, R11
	DECQ AX
	JNZ  mulGFNI_6x9_64Xor_loop
	VZEROUPPER

mulGFNI_6x9_64Xor_end:
	RET

// func mulAvxGFNI_6x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x9Xor(SB), $0-88
	// Loading 5 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulAvxGFNI_6x9Xor_loop:
	// Load 9 outputs
	MOVQ    (R10), R12
	VMOVDQU (R12)(R11*1), Y5
	MOVQ    24(R10), R12
	VMOVDQU (R12)(R11*1), Y6
	MOVQ    48(R10), R12
	VMOVDQU (R12)(R11*1), Y7
	MOVQ    72(R10), R12
	VMOVDQU (R12)(R11*1), Y8
	MOVQ    96(R10), R12
	VMOVDQU (R12)(R11*1), Y9
	MOVQ    120(R10), R12
	VMOVDQU (R12)(R11*1), Y10
	MOVQ    144(R10), R12
	VMOVDQU (R12)(R11*1), Y11
	MOVQ    168(R10), R12
	VMOVDQU (R12)(R11*1), Y12
	MOVQ    192(R10), R12
	VMOVDQU (R12)(R11*1), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y9, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y10, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y11, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y12, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y13, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxGFNI_6x9Xor_loop
	VZEROUPPER

mulAvxGFNI_6x9Xor_end:
	RET

// func mulAvxTwo_6x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x9Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 122 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  start+72(FP), R11

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_6x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	MOVQ    (R10), R12
	VMOVDQU (R12)(R11*1), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	MOVQ    24(R10), R12
	VMOVDQU (R12)(R11*1), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	MOVQ    48(R10), R12
	VMOVDQU (R12)(R11*1), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	MOVQ    72(R10), R12
	VMOVDQU (R12)(R11*1), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	MOVQ    96(R10), R12
	VMOVDQU (R12)(R11*1), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	MOVQ    120(R10), R12
	VMOVDQU (R12)(R11*1), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	MOVQ    144(R10), R12
	VMOVDQU (R12)(R11*1), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	MOVQ    168(R10), R12
	VMOVDQU (R12)(R11*1), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	MOVQ    192(R10), R12
	VMOVDQU (R12)(R11*1), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R10), R12
	VMOVDQU Y0, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y1, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y2, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y3, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y8, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxTwo_6x9Xor_loop
	VZEROUPPER

mulAvxTwo_6x9Xor_end:
	RET

// func mulAvxTwo_6x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x10(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 135 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  start+72(FP), R11

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_6x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R10), R12
	VMOVDQU Y0, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y1, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y2, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y3, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    216(R10), R12
	VMOVDQU Y9, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxTwo_6x10_loop
	VZEROUPPER

mulAvxTwo_6x10_end:
	RET

// func mulGFNI_6x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x10_64(SB), $0-88
	// Loading 20 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulGFNI_6x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R10), R12
	VMOVDQU64 Z20, (R12)(R11*1)
	MOVQ      24(R10), R12
	VMOVDQU64 Z21, (R12)(R11*1)
	MOVQ      48(R10), R12
	VMOVDQU64 Z22, (R12)(R11*1)
	MOVQ      72(R10), R12
	VMOVDQU64 Z23, (R12)(R11*1)
	MOVQ      96(R10), R12
	VMOVDQU64 Z24, (R12)(R11*1)
	MOVQ      120(R10), R12
	VMOVDQU64 Z25, (R12)(R11*1)
	MOVQ      144(R10), R12
	VMOVDQU64 Z26, (R12)(R11*1)
	MOVQ      168(R10), R12
	VMOVDQU64 Z27, (R12)(R11*1)
	MOVQ      192(R10), R12
	VMOVDQU64 Z28, (R12)(R11*1)
	MOVQ      216(R10), R12
	VMOVDQU64 Z29, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x40, R11
	DECQ AX
	JNZ  mulGFNI_6x10_64_loop
	VZEROUPPER

mulGFNI_6x10_64_end:
	RET

// func mulAvxGFNI_6x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x10(SB), $0-88
	// Loading 4 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulAvxGFNI_6x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y9, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y10, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y11, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y12, (R12)(R11*1)
	MOVQ    216(R10), R12
	VMOVDQU Y13, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxGFNI_6x10_loop
	VZEROUPPER

mulAvxGFNI_6x10_end:
	RET

// func mulGFNI_6x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_6x10_64Xor(SB), $0-88
	// Loading 20 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_6x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), DX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulGFNI_6x10_64Xor_loop:
	// Load 10 outputs
	MOVQ      (R10), R12
	VMOVDQU64 (R12)(R11*1), Z20
	MOVQ      24(R10), R12
	VMOVDQU64 (R12)(R11*1), Z21
	MOVQ      48(R10), R12
	VMOVDQU64 (R12)(R11*1), Z22
	MOVQ      72(R10), R12
	VMOVDQU64 (R12)(R11*1), Z23
	MOVQ      96(R10), R12
	VMOVDQU64 (R12)(R11*1), Z24
	MOVQ      120(R10), R12
	VMOVDQU64 (R12)(R11*1), Z25
	MOVQ      144(R10), R12
	VMOVDQU64 (R12)(R11*1), Z26
	MOVQ      168(R10), R12
	VMOVDQU64 (R12)(R11*1), Z27
	MOVQ      192(R10), R12
	VMOVDQU64 (R12)(R11*1), Z28
	MOVQ      216(R10), R12
	VMOVDQU64 (R12)(R11*1), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R10), R12
	VMOVDQU64 Z20, (R12)(R11*1)
	MOVQ      24(R10), R12
	VMOVDQU64 Z21, (R12)(R11*1)
	MOVQ      48(R10), R12
	VMOVDQU64 Z22, (R12)(R11*1)
	MOVQ      72(R10), R12
	VMOVDQU64 Z23, (R12)(R11*1)
	MOVQ      96(R10), R12
	VMOVDQU64 Z24, (R12)(R11*1)
	MOVQ      120(R10), R12
	VMOVDQU64 Z25, (R12)(R11*1)
	MOVQ      144(R10), R12
	VMOVDQU64 Z26, (R12)(R11*1)
	MOVQ      168(R10), R12
	VMOVDQU64 Z27, (R12)(R11*1)
	MOVQ      192(R10), R12
	VMOVDQU64 Z28, (R12)(R11*1)
	MOVQ      216(R10), R12
	VMOVDQU64 Z29, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x40, R11
	DECQ AX
	JNZ  mulGFNI_6x10_64Xor_loop
	VZEROUPPER

mulGFNI_6x10_64Xor_end:
	RET

// func mulAvxGFNI_6x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_6x10Xor(SB), $0-88
	// Loading 4 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_6x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), DX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         start+72(FP), R11

	// Add start offset to input
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, DX

mulAvxGFNI_6x10Xor_loop:
	// Load 10 outputs
	MOVQ    (R10), R12
	VMOVDQU (R12)(R11*1), Y4
	MOVQ    24(R10), R12
	VMOVDQU (R12)(R11*1), Y5
	MOVQ    48(R10), R12
	VMOVDQU (R12)(R11*1), Y6
	MOVQ    72(R10), R12
	VMOVDQU (R12)(R11*1), Y7
	MOVQ    96(R10), R12
	VMOVDQU (R12)(R11*1), Y8
	MOVQ    120(R10), R12
	VMOVDQU (R12)(R11*1), Y9
	MOVQ    144(R10), R12
	VMOVDQU (R12)(R11*1), Y10
	MOVQ    168(R10), R12
	VMOVDQU (R12)(R11*1), Y11
	MOVQ    192(R10), R12
	VMOVDQU (R12)(R11*1), Y12
	MOVQ    216(R10), R12
	VMOVDQU (R12)(R11*1), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y9, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y10, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y11, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y12, (R12)(R11*1)
	MOVQ    216(R10), R12
	VMOVDQU Y13, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxGFNI_6x10Xor_loop
	VZEROUPPER

mulAvxGFNI_6x10Xor_end:
	RET

// func mulAvxTwo_6x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_6x10Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 135 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_6x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), DX
	MOVQ  out_base+48(FP), R10
	MOVQ  start+72(FP), R11

	// Add start offset to input
	ADDQ         R11, BX
	ADDQ         R11, SI
	ADDQ         R11, DI
	ADDQ         R11, R8
	ADDQ         R11, R9
	ADDQ         R11, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_6x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	MOVQ    (R10), R12
	VMOVDQU (R12)(R11*1), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	MOVQ    24(R10), R12
	VMOVDQU (R12)(R11*1), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	MOVQ    48(R10), R12
	VMOVDQU (R12)(R11*1), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	MOVQ    72(R10), R12
	VMOVDQU (R12)(R11*1), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	MOVQ    96(R10), R12
	VMOVDQU (R12)(R11*1), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	MOVQ    120(R10), R12
	VMOVDQU (R12)(R11*1), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	MOVQ    144(R10), R12
	VMOVDQU (R12)(R11*1), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	MOVQ    168(R10), R12
	VMOVDQU (R12)(R11*1), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	MOVQ    192(R10), R12
	VMOVDQU (R12)(R11*1), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	MOVQ    216(R10), R12
	VMOVDQU (R12)(R11*1), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R10), R12
	VMOVDQU Y0, (R12)(R11*1)
	MOVQ    24(R10), R12
	VMOVDQU Y1, (R12)(R11*1)
	MOVQ    48(R10), R12
	VMOVDQU Y2, (R12)(R11*1)
	MOVQ    72(R10), R12
	VMOVDQU Y3, (R12)(R11*1)
	MOVQ    96(R10), R12
	VMOVDQU Y4, (R12)(R11*1)
	MOVQ    120(R10), R12
	VMOVDQU Y5, (R12)(R11*1)
	MOVQ    144(R10), R12
	VMOVDQU Y6, (R12)(R11*1)
	MOVQ    168(R10), R12
	VMOVDQU Y7, (R12)(R11*1)
	MOVQ    192(R10), R12
	VMOVDQU Y8, (R12)(R11*1)
	MOVQ    216(R10), R12
	VMOVDQU Y9, (R12)(R11*1)

	// Prepare for next loop
	ADDQ $0x20, R11
	DECQ AX
	JNZ  mulAvxTwo_6x10Xor_loop
	VZEROUPPER

mulAvxTwo_6x10Xor_end:
	RET

// func mulAvxTwo_7x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R11
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_7x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R11)
	VMOVDQU Y1, 32(R11)
	ADDQ    $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x1_64_loop
	VZEROUPPER

mulAvxTwo_7x1_64_end:
	RET

// func mulGFNI_7x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), CX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R10
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, CX

mulGFNI_7x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z8
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z8, Z7

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z8
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z8
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z8
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z8
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z8
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (CX), Z8
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Store 1 outputs
	VMOVDQU64 Z7, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x1_64_loop
	VZEROUPPER

mulGFNI_7x1_64_end:
	RET

// func mulAvxGFNI_7x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), CX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R10
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, CX

mulAvxGFNI_7x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y8
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y8, Y7

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y8
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y8
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y8
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y8
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y8
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (CX), Y8
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y6, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Store 1 outputs
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x1_loop
	VZEROUPPER

mulAvxGFNI_7x1_end:
	RET

// func mulGFNI_7x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), CX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R10
	MOVQ            start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, CX

mulGFNI_7x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (R10), Z7

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z8
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z8
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z8
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z8
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z8
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z8
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (CX), Z8
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z6, Z8, Z8
	VXORPD         Z7, Z8, Z7

	// Store 1 outputs
	VMOVDQU64 Z7, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x1_64Xor_loop
	VZEROUPPER

mulGFNI_7x1_64Xor_end:
	RET

// func mulAvxGFNI_7x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 10 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), CX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R10
	MOVQ         start+72(FP), R11

	// Add start offset to output
	ADDQ R11, R10

	// Add start offset to input
	ADDQ R11, DX
	ADDQ R11, BX
	ADDQ R11, SI
	ADDQ R11, DI
	ADDQ R11, R8
	ADDQ R11, R9
	ADDQ R11, CX

mulAvxGFNI_7x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (R10), Y7

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y8
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y8
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y8
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y8
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y8
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y8
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (CX), Y8
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y6, Y8, Y8
	VXORPD         Y7, Y8, Y7

	// Store 1 outputs
	VMOVDQU Y7, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x1Xor_loop
	VZEROUPPER

mulAvxGFNI_7x1Xor_end:
	RET

// func mulAvxTwo_7x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R11
	MOVQ  start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R12
	MOVQ         R12, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_7x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (R11), Y0
	VMOVDQU 32(R11), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R11)
	VMOVDQU Y1, 32(R11)
	ADDQ    $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_7x1_64Xor_end:
	RET

// func mulAvxTwo_7x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 65 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R11
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12
	ADDQ R13, R11

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_7x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R12)
	VMOVDQU Y1, 32(R12)
	ADDQ    $0x40, R12
	VMOVDQU Y2, (R11)
	VMOVDQU Y3, 32(R11)
	ADDQ    $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x2_64_loop
	VZEROUPPER

mulAvxTwo_7x2_64_end:
	RET

// func mulGFNI_7x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), CX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R10
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11
	ADDQ R12, R10

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, CX

mulGFNI_7x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z16
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z16, Z14
	VGF2P8AFFINEQB $0x00, Z1, Z16, Z15

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z16
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z16
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z16
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z16
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z16
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (CX), Z16
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z13, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Store 2 outputs
	VMOVDQU64 Z14, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z15, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x2_64_loop
	VZEROUPPER

mulGFNI_7x2_64_end:
	RET

// func mulAvxGFNI_7x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x2(SB), $0-88
	// Loading 12 of 14 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R11
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12
	ADDQ R13, R11

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, DX

mulAvxGFNI_7x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x2_loop
	VZEROUPPER

mulAvxGFNI_7x2_end:
	RET

// func mulGFNI_7x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), CX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R10
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11
	ADDQ R12, R10

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, CX

mulGFNI_7x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (R11), Z14
	VMOVDQU64 (R10), Z15

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z16
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z1, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z16
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z16
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z16
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z16
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z16
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (CX), Z16
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z12, Z16, Z17
	VXORPD         Z14, Z17, Z14
	VGF2P8AFFINEQB $0x00, Z13, Z16, Z17
	VXORPD         Z15, Z17, Z15

	// Store 2 outputs
	VMOVDQU64 Z14, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z15, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x2_64Xor_loop
	VZEROUPPER

mulGFNI_7x2_64Xor_end:
	RET

// func mulAvxGFNI_7x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x2Xor(SB), $0-88
	// Loading 12 of 14 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 18 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R11
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12
	ADDQ R13, R11

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, DX

mulAvxGFNI_7x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (R12), Y12
	VMOVDQU (R11), Y13

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x2Xor_loop
	VZEROUPPER

mulAvxGFNI_7x2Xor_end:
	RET

// func mulAvxTwo_7x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 65 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R11
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12
	ADDQ R13, R11

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_7x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R12), Y0
	VMOVDQU 32(R12), Y1
	VMOVDQU (R11), Y2
	VMOVDQU 32(R11), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R12)
	VMOVDQU Y1, 32(R12)
	ADDQ    $0x40, R12
	VMOVDQU Y2, (R11)
	VMOVDQU Y3, 32(R11)
	ADDQ    $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_7x2_64Xor_end:
	RET

// func mulAvxTwo_7x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 94 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R11
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R11

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_7x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R12)
	VMOVDQU Y1, 32(R12)
	ADDQ    $0x40, R12
	VMOVDQU Y2, (R13)
	VMOVDQU Y3, 32(R13)
	ADDQ    $0x40, R13
	VMOVDQU Y4, (R11)
	VMOVDQU Y5, 32(R11)
	ADDQ    $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x3_64_loop
	VZEROUPPER

mulAvxTwo_7x3_64_end:
	RET

// func mulGFNI_7x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), CX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R10
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R10

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, CX

mulGFNI_7x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z23

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z24
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z24
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z24
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z24
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z24
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z20, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 3 outputs
	VMOVDQU64 Z21, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z22, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x3_64_loop
	VZEROUPPER

mulGFNI_7x3_64_end:
	RET

// func mulAvxGFNI_7x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x3(SB), $0-88
	// Loading 11 of 21 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R11
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R11

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, DX

mulAvxGFNI_7x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x3_loop
	VZEROUPPER

mulAvxGFNI_7x3_end:
	RET

// func mulGFNI_7x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), CX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R10
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R11
	ADDQ R13, R12
	ADDQ R13, R10

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, CX

mulGFNI_7x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (R11), Z21
	VMOVDQU64 (R12), Z22
	VMOVDQU64 (R10), Z23

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z24
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z24
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z24
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z24
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z24
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z24
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (CX), Z24
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
	VXORPD         Z21, Z25, Z21
	VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
	VXORPD         Z22, Z25, Z22
	VGF2P8AFFINEQB $0x00, Z20, Z24, Z25
	VXORPD         Z23, Z25, Z23

	// Store 3 outputs
	VMOVDQU64 Z21, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z22, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z23, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x3_64Xor_loop
	VZEROUPPER

mulGFNI_7x3_64Xor_end:
	RET

// func mulAvxGFNI_7x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x3Xor(SB), $0-88
	// Loading 11 of 21 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 26 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R11
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R11

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, DX

mulAvxGFNI_7x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (R12), Y11
	VMOVDQU (R13), Y12
	VMOVDQU (R11), Y13

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x3Xor_loop
	VZEROUPPER

mulAvxGFNI_7x3Xor_end:
	RET

// func mulAvxTwo_7x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 94 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R11
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R11

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_7x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R12), Y0
	VMOVDQU 32(R12), Y1
	VMOVDQU (R13), Y2
	VMOVDQU 32(R13), Y3
	VMOVDQU (R11), Y4
	VMOVDQU 32(R11), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R12)
	VMOVDQU Y1, 32(R12)
	ADDQ    $0x40, R12
	VMOVDQU Y2, (R13)
	VMOVDQU Y3, 32(R13)
	ADDQ    $0x40, R13
	VMOVDQU Y4, (R11)
	VMOVDQU Y5, 32(R11)
	ADDQ    $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_7x3_64Xor_end:
	RET

// func mulAvxTwo_7x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x4(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 65 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R14
	MOVQ  72(R11), R11
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R11

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_7x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y1, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y2, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x4_loop
	VZEROUPPER

mulAvxTwo_7x4_end:
	RET

// func mulGFNI_7x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x4_64(SB), $0-88
	// Loading 26 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R14
	MOVQ            72(R11), R11
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R11

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, DX

mulGFNI_7x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x4_64_loop
	VZEROUPPER

mulGFNI_7x4_64_end:
	RET

// func mulAvxGFNI_7x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x4(SB), $0-88
	// Loading 10 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R14
	MOVQ         72(R11), R11
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R11

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, DX

mulAvxGFNI_7x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x4_loop
	VZEROUPPER

mulAvxGFNI_7x4_end:
	RET

// func mulGFNI_7x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x4_64Xor(SB), $0-88
	// Loading 26 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R14
	MOVQ            72(R11), R11
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R11

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, DX

mulGFNI_7x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (R12), Z26
	VMOVDQU64 (R13), Z27
	VMOVDQU64 (R14), Z28
	VMOVDQU64 (R11), Z29

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	VMOVDQU64 Z26, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x4_64Xor_loop
	VZEROUPPER

mulGFNI_7x4_64Xor_end:
	RET

// func mulAvxGFNI_7x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x4Xor(SB), $0-88
	// Loading 10 of 28 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 34 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R14
	MOVQ         72(R11), R11
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R11

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, DX

mulAvxGFNI_7x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (R12), Y10
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (R11), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x4Xor_loop
	VZEROUPPER

mulAvxGFNI_7x4Xor_end:
	RET

// func mulAvxTwo_7x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x4Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 65 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R14
	MOVQ  72(R11), R11
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R11

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_7x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (R12), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R13), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R14), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (R11), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y1, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y2, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y3, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x4Xor_loop
	VZEROUPPER

mulAvxTwo_7x4Xor_end:
	RET

// func mulAvxTwo_7x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x5(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 80 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R14
	MOVQ  72(R11), R15
	MOVQ  96(R11), R11
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_7x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y1, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y2, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y3, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x5_loop
	VZEROUPPER

mulAvxTwo_7x5_end:
	RET

// func mulGFNI_7x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x5_64(SB), $8-88
	// Loading 25 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R14
	MOVQ            72(R11), R15
	MOVQ            96(R11), R11
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, DX

mulGFNI_7x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x5_64_loop
	VZEROUPPER

mulGFNI_7x5_64_end:
	RET

// func mulAvxGFNI_7x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x5(SB), $8-88
	// Loading 9 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R14
	MOVQ         72(R11), R15
	MOVQ         96(R11), R11
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, DX

mulAvxGFNI_7x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x5_loop
	VZEROUPPER

mulAvxGFNI_7x5_end:
	RET

// func mulGFNI_7x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x5_64Xor(SB), $8-88
	// Loading 25 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R14
	MOVQ            72(R11), R15
	MOVQ            96(R11), R11
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, DX

mulGFNI_7x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R11), Z29

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_7x5_64Xor_loop
	VZEROUPPER

mulGFNI_7x5_64Xor_end:
	RET

// func mulAvxGFNI_7x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x5Xor(SB), $8-88
	// Loading 9 of 35 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R14
	MOVQ         72(R11), R15
	MOVQ         96(R11), R11
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, DX

mulAvxGFNI_7x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R11), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_7x5Xor_loop
	VZEROUPPER

mulAvxGFNI_7x5Xor_end:
	RET

// func mulAvxTwo_7x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x5Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 80 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R14
	MOVQ  72(R11), R15
	MOVQ  96(R11), R11
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_7x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (R12), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (R13), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R14), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R15), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y1, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y2, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y3, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_7x5Xor_loop
	VZEROUPPER

mulAvxTwo_7x5Xor_end:
	RET

// func mulAvxTwo_7x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x6(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 95 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x6_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), AX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R14
	MOVQ  96(R10), R15
	MOVQ  120(R10), R10
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_7x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (AX), Y9
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y4, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y5, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_7x6_loop
	VZEROUPPER

mulAvxTwo_7x6_end:
	RET

// func mulGFNI_7x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x6_64(SB), $8-88
	// Loading 24 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), AX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R13
	MOVQ            72(R10), R14
	MOVQ            96(R10), R15
	MOVQ            120(R10), R10
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_7x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_7x6_64_loop
	VZEROUPPER

mulGFNI_7x6_64_end:
	RET

// func mulAvxGFNI_7x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x6(SB), $8-88
	// Loading 8 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), AX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R14
	MOVQ         96(R10), R15
	MOVQ         120(R10), R10
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_7x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_7x6_loop
	VZEROUPPER

mulAvxGFNI_7x6_end:
	RET

// func mulGFNI_7x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x6_64Xor(SB), $8-88
	// Loading 24 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), AX
	MOVQ            out_base+48(FP), R10
	MOVQ            out_base+48(FP), R10
	MOVQ            (R10), R11
	MOVQ            24(R10), R12
	MOVQ            48(R10), R13
	MOVQ            72(R10), R14
	MOVQ            96(R10), R15
	MOVQ            120(R10), R10
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_7x6_64Xor_loop:
	// Load 6 outputs
	VMOVDQU64 (R11), Z24
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R10), Z29

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	VMOVDQU64 Z24, (R11)
	ADDQ      $0x40, R11
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R10)
	ADDQ      $0x40, R10

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_7x6_64Xor_loop
	VZEROUPPER

mulGFNI_7x6_64Xor_end:
	RET

// func mulAvxGFNI_7x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x6Xor(SB), $8-88
	// Loading 8 of 42 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 50 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), AX
	MOVQ         out_base+48(FP), R10
	MOVQ         out_base+48(FP), R10
	MOVQ         (R10), R11
	MOVQ         24(R10), R12
	MOVQ         48(R10), R13
	MOVQ         72(R10), R14
	MOVQ         96(R10), R15
	MOVQ         120(R10), R10
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_7x6Xor_loop:
	// Load 6 outputs
	VMOVDQU (R11), Y8
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R10), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_7x6Xor_loop
	VZEROUPPER

mulAvxGFNI_7x6Xor_end:
	RET

// func mulAvxTwo_7x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x6Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 95 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x6Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), AX
	MOVQ  out_base+48(FP), R10
	MOVQ  (R10), R11
	MOVQ  24(R10), R12
	MOVQ  48(R10), R13
	MOVQ  72(R10), R14
	MOVQ  96(R10), R15
	MOVQ  120(R10), R10
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R10

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_7x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (R11), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU (R12), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU (R13), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU (R14), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU (R15), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU (R10), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (AX), Y9
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	VMOVDQU Y0, (R11)
	ADDQ    $0x20, R11
	VMOVDQU Y1, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y2, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y3, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y4, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y5, (R10)
	ADDQ    $0x20, R10

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_7x6Xor_loop
	VZEROUPPER

mulAvxTwo_7x6Xor_end:
	RET

// func mulAvxTwo_7x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x7(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 110 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_7x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x7_loop
	VZEROUPPER

mulAvxTwo_7x7_end:
	RET

// func mulGFNI_7x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x7_64(SB), $0-88
	// Loading 23 of 49 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x7_64_loop
	VZEROUPPER

mulGFNI_7x7_64_end:
	RET

// func mulAvxGFNI_7x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x7(SB), $0-88
	// Loading 7 of 49 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x7_loop
	VZEROUPPER

mulAvxGFNI_7x7_end:
	RET

// func mulGFNI_7x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x7_64Xor(SB), $0-88
	// Loading 23 of 49 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x7_64Xor_loop:
	// Load 7 outputs
	MOVQ      (R11), R13
	VMOVDQU64 (R13)(R12*1), Z23
	MOVQ      24(R11), R13
	VMOVDQU64 (R13)(R12*1), Z24
	MOVQ      48(R11), R13
	VMOVDQU64 (R13)(R12*1), Z25
	MOVQ      72(R11), R13
	VMOVDQU64 (R13)(R12*1), Z26
	MOVQ      96(R11), R13
	VMOVDQU64 (R13)(R12*1), Z27
	MOVQ      120(R11), R13
	VMOVDQU64 (R13)(R12*1), Z28
	MOVQ      144(R11), R13
	VMOVDQU64 (R13)(R12*1), Z29

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x7_64Xor_loop
	VZEROUPPER

mulGFNI_7x7_64Xor_end:
	RET

// func mulAvxGFNI_7x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x7Xor(SB), $0-88
	// Loading 7 of 49 tables to registers
	// Destination kept on stack
	// Full registers estimated 58 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x7Xor_loop:
	// Load 7 outputs
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y7
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y8
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y9
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y10
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y11
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y12
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x7Xor_loop
	VZEROUPPER

mulAvxGFNI_7x7Xor_end:
	RET

// func mulAvxTwo_7x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x7Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 110 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_7x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x7Xor_loop
	VZEROUPPER

mulAvxTwo_7x7Xor_end:
	RET

// func mulAvxTwo_7x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x8(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 125 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_7x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y7, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x8_loop
	VZEROUPPER

mulAvxTwo_7x8_end:
	RET

// func mulGFNI_7x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x8_64(SB), $0-88
	// Loading 22 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 66 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z22, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      168(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x8_64_loop
	VZEROUPPER

mulGFNI_7x8_64_end:
	RET

// func mulAvxGFNI_7x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x8(SB), $0-88
	// Loading 6 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 66 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x8_loop
	VZEROUPPER

mulAvxGFNI_7x8_end:
	RET

// func mulGFNI_7x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x8_64Xor(SB), $0-88
	// Loading 22 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 66 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x8_64Xor_loop:
	// Load 8 outputs
	MOVQ      (R11), R13
	VMOVDQU64 (R13)(R12*1), Z22
	MOVQ      24(R11), R13
	VMOVDQU64 (R13)(R12*1), Z23
	MOVQ      48(R11), R13
	VMOVDQU64 (R13)(R12*1), Z24
	MOVQ      72(R11), R13
	VMOVDQU64 (R13)(R12*1), Z25
	MOVQ      96(R11), R13
	VMOVDQU64 (R13)(R12*1), Z26
	MOVQ      120(R11), R13
	VMOVDQU64 (R13)(R12*1), Z27
	MOVQ      144(R11), R13
	VMOVDQU64 (R13)(R12*1), Z28
	MOVQ      168(R11), R13
	VMOVDQU64 (R13)(R12*1), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z22, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      168(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x8_64Xor_loop
	VZEROUPPER

mulGFNI_7x8_64Xor_end:
	RET

// func mulAvxGFNI_7x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x8Xor(SB), $0-88
	// Loading 6 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 66 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x8Xor_loop:
	// Load 8 outputs
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y6
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y7
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y8
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y9
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y10
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y11
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y12
	MOVQ    168(R11), R13
	VMOVDQU (R13)(R12*1), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x8Xor_loop
	VZEROUPPER

mulAvxGFNI_7x8Xor_end:
	RET

// func mulAvxTwo_7x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x8Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 125 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_7x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	MOVQ    168(R11), R13
	VMOVDQU (R13)(R12*1), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y7, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x8Xor_loop
	VZEROUPPER

mulAvxTwo_7x8Xor_end:
	RET

// func mulAvxTwo_7x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x9(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 140 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_7x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y8, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x9_loop
	VZEROUPPER

mulAvxTwo_7x9_end:
	RET

// func mulGFNI_7x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x9_64(SB), $0-88
	// Loading 21 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z21, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z22, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      168(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      192(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x9_64_loop
	VZEROUPPER

mulGFNI_7x9_64_end:
	RET

// func mulAvxGFNI_7x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x9(SB), $0-88
	// Loading 5 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x9_loop
	VZEROUPPER

mulAvxGFNI_7x9_end:
	RET

// func mulGFNI_7x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x9_64Xor(SB), $0-88
	// Loading 21 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x9_64Xor_loop:
	// Load 9 outputs
	MOVQ      (R11), R13
	VMOVDQU64 (R13)(R12*1), Z21
	MOVQ      24(R11), R13
	VMOVDQU64 (R13)(R12*1), Z22
	MOVQ      48(R11), R13
	VMOVDQU64 (R13)(R12*1), Z23
	MOVQ      72(R11), R13
	VMOVDQU64 (R13)(R12*1), Z24
	MOVQ      96(R11), R13
	VMOVDQU64 (R13)(R12*1), Z25
	MOVQ      120(R11), R13
	VMOVDQU64 (R13)(R12*1), Z26
	MOVQ      144(R11), R13
	VMOVDQU64 (R13)(R12*1), Z27
	MOVQ      168(R11), R13
	VMOVDQU64 (R13)(R12*1), Z28
	MOVQ      192(R11), R13
	VMOVDQU64 (R13)(R12*1), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z21, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z22, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      168(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      192(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x9_64Xor_loop
	VZEROUPPER

mulGFNI_7x9_64Xor_end:
	RET

// func mulAvxGFNI_7x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x9Xor(SB), $0-88
	// Loading 5 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x9Xor_loop:
	// Load 9 outputs
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y5
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y6
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y7
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y8
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y9
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y10
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y11
	MOVQ    168(R11), R13
	VMOVDQU (R13)(R12*1), Y12
	MOVQ    192(R11), R13
	VMOVDQU (R13)(R12*1), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x9Xor_loop
	VZEROUPPER

mulAvxGFNI_7x9Xor_end:
	RET

// func mulAvxTwo_7x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x9Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 140 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_7x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	MOVQ    168(R11), R13
	VMOVDQU (R13)(R12*1), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	MOVQ    192(R11), R13
	VMOVDQU (R13)(R12*1), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y8, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x9Xor_loop
	VZEROUPPER

mulAvxTwo_7x9Xor_end:
	RET

// func mulAvxTwo_7x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x10(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 155 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_7x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    216(R11), R13
	VMOVDQU Y9, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x10_loop
	VZEROUPPER

mulAvxTwo_7x10_end:
	RET

// func mulGFNI_7x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x10_64(SB), $0-88
	// Loading 20 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z20, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z21, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z22, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      168(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      192(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      216(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x10_64_loop
	VZEROUPPER

mulGFNI_7x10_64_end:
	RET

// func mulAvxGFNI_7x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x10(SB), $0-88
	// Loading 4 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    216(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x10_loop
	VZEROUPPER

mulAvxGFNI_7x10_end:
	RET

// func mulGFNI_7x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_7x10_64Xor(SB), $0-88
	// Loading 20 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_7x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), DX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulGFNI_7x10_64Xor_loop:
	// Load 10 outputs
	MOVQ      (R11), R13
	VMOVDQU64 (R13)(R12*1), Z20
	MOVQ      24(R11), R13
	VMOVDQU64 (R13)(R12*1), Z21
	MOVQ      48(R11), R13
	VMOVDQU64 (R13)(R12*1), Z22
	MOVQ      72(R11), R13
	VMOVDQU64 (R13)(R12*1), Z23
	MOVQ      96(R11), R13
	VMOVDQU64 (R13)(R12*1), Z24
	MOVQ      120(R11), R13
	VMOVDQU64 (R13)(R12*1), Z25
	MOVQ      144(R11), R13
	VMOVDQU64 (R13)(R12*1), Z26
	MOVQ      168(R11), R13
	VMOVDQU64 (R13)(R12*1), Z27
	MOVQ      192(R11), R13
	VMOVDQU64 (R13)(R12*1), Z28
	MOVQ      216(R11), R13
	VMOVDQU64 (R13)(R12*1), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R11), R13
	VMOVDQU64 Z20, (R13)(R12*1)
	MOVQ      24(R11), R13
	VMOVDQU64 Z21, (R13)(R12*1)
	MOVQ      48(R11), R13
	VMOVDQU64 Z22, (R13)(R12*1)
	MOVQ      72(R11), R13
	VMOVDQU64 Z23, (R13)(R12*1)
	MOVQ      96(R11), R13
	VMOVDQU64 Z24, (R13)(R12*1)
	MOVQ      120(R11), R13
	VMOVDQU64 Z25, (R13)(R12*1)
	MOVQ      144(R11), R13
	VMOVDQU64 Z26, (R13)(R12*1)
	MOVQ      168(R11), R13
	VMOVDQU64 Z27, (R13)(R12*1)
	MOVQ      192(R11), R13
	VMOVDQU64 Z28, (R13)(R12*1)
	MOVQ      216(R11), R13
	VMOVDQU64 Z29, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x40, R12
	DECQ AX
	JNZ  mulGFNI_7x10_64Xor_loop
	VZEROUPPER

mulGFNI_7x10_64Xor_end:
	RET

// func mulAvxGFNI_7x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_7x10Xor(SB), $0-88
	// Loading 4 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_7x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), DX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         start+72(FP), R12

	// Add start offset to input
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, DX

mulAvxGFNI_7x10Xor_loop:
	// Load 10 outputs
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y4
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y5
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y6
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y7
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y8
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y9
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y10
	MOVQ    168(R11), R13
	VMOVDQU (R13)(R12*1), Y11
	MOVQ    192(R11), R13
	VMOVDQU (R13)(R12*1), Y12
	MOVQ    216(R11), R13
	VMOVDQU (R13)(R12*1), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y9, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y10, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y11, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y12, (R13)(R12*1)
	MOVQ    216(R11), R13
	VMOVDQU Y13, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxGFNI_7x10Xor_loop
	VZEROUPPER

mulAvxGFNI_7x10Xor_end:
	RET

// func mulAvxTwo_7x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_7x10Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 155 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_7x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), DX
	MOVQ  out_base+48(FP), R11
	MOVQ  start+72(FP), R12

	// Add start offset to input
	ADDQ         R12, BX
	ADDQ         R12, SI
	ADDQ         R12, DI
	ADDQ         R12, R8
	ADDQ         R12, R9
	ADDQ         R12, R10
	ADDQ         R12, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_7x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	MOVQ    (R11), R13
	VMOVDQU (R13)(R12*1), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	MOVQ    24(R11), R13
	VMOVDQU (R13)(R12*1), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	MOVQ    48(R11), R13
	VMOVDQU (R13)(R12*1), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	MOVQ    72(R11), R13
	VMOVDQU (R13)(R12*1), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	MOVQ    96(R11), R13
	VMOVDQU (R13)(R12*1), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	MOVQ    120(R11), R13
	VMOVDQU (R13)(R12*1), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	MOVQ    144(R11), R13
	VMOVDQU (R13)(R12*1), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	MOVQ    168(R11), R13
	VMOVDQU (R13)(R12*1), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	MOVQ    192(R11), R13
	VMOVDQU (R13)(R12*1), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	MOVQ    216(R11), R13
	VMOVDQU (R13)(R12*1), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R11), R13
	VMOVDQU Y0, (R13)(R12*1)
	MOVQ    24(R11), R13
	VMOVDQU Y1, (R13)(R12*1)
	MOVQ    48(R11), R13
	VMOVDQU Y2, (R13)(R12*1)
	MOVQ    72(R11), R13
	VMOVDQU Y3, (R13)(R12*1)
	MOVQ    96(R11), R13
	VMOVDQU Y4, (R13)(R12*1)
	MOVQ    120(R11), R13
	VMOVDQU Y5, (R13)(R12*1)
	MOVQ    144(R11), R13
	VMOVDQU Y6, (R13)(R12*1)
	MOVQ    168(R11), R13
	VMOVDQU Y7, (R13)(R12*1)
	MOVQ    192(R11), R13
	VMOVDQU Y8, (R13)(R12*1)
	MOVQ    216(R11), R13
	VMOVDQU Y9, (R13)(R12*1)

	// Prepare for next loop
	ADDQ $0x20, R12
	DECQ AX
	JNZ  mulAvxTwo_7x10Xor_loop
	VZEROUPPER

mulAvxTwo_7x10Xor_end:
	RET

// func mulAvxTwo_8x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R12
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_8x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (R11), Y6
	VMOVDQU 32(R11), Y5
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 448(CX), Y3
	VMOVDQU 480(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R12)
	VMOVDQU Y1, 32(R12)
	ADDQ    $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x1_64_loop
	VZEROUPPER

mulAvxTwo_8x1_64_end:
	RET

// func mulGFNI_8x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), CX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R11
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, CX

mulGFNI_8x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z9
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z9, Z8

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z9
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z9
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z9
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z9
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z9
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (R10), Z9
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z6, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU64      (CX), Z9
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z7, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Store 1 outputs
	VMOVDQU64 Z8, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x1_64_loop
	VZEROUPPER

mulGFNI_8x1_64_end:
	RET

// func mulAvxGFNI_8x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), R10
	MOVQ         168(CX), CX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R11
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, CX

mulAvxGFNI_8x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y9
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y9, Y8

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y9
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y9
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y9
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y9
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y9
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (R10), Y9
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y6, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 7 to 1 outputs
	VMOVDQU        (CX), Y9
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y7, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Store 1 outputs
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x1_loop
	VZEROUPPER

mulAvxGFNI_8x1_end:
	RET

// func mulGFNI_8x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), CX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R11
	MOVQ            start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, CX

mulGFNI_8x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (R11), Z8

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z9
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z9
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z9
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z9
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z9
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z9
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (R10), Z9
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z6, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU64      (CX), Z9
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z7, Z9, Z9
	VXORPD         Z8, Z9, Z8

	// Store 1 outputs
	VMOVDQU64 Z8, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x1_64Xor_loop
	VZEROUPPER

mulGFNI_8x1_64Xor_end:
	RET

// func mulAvxGFNI_8x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 11 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), R10
	MOVQ         168(CX), CX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R11
	MOVQ         start+72(FP), R12

	// Add start offset to output
	ADDQ R12, R11

	// Add start offset to input
	ADDQ R12, DX
	ADDQ R12, BX
	ADDQ R12, SI
	ADDQ R12, DI
	ADDQ R12, R8
	ADDQ R12, R9
	ADDQ R12, R10
	ADDQ R12, CX

mulAvxGFNI_8x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (R11), Y8

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y9
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y9
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y9
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y9
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y9
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y9
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (R10), Y9
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y6, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Load and process 32 bytes from input 7 to 1 outputs
	VMOVDQU        (CX), Y9
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y7, Y9, Y9
	VXORPD         Y8, Y9, Y8

	// Store 1 outputs
	VMOVDQU Y8, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x1Xor_loop
	VZEROUPPER

mulAvxGFNI_8x1Xor_end:
	RET

// func mulAvxTwo_8x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R12
	MOVQ  start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R13
	MOVQ         R13, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_8x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (R12), Y0
	VMOVDQU 32(R12), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (R11), Y6
	VMOVDQU 32(R11), Y5
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 448(CX), Y3
	VMOVDQU 480(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R12)
	VMOVDQU Y1, 32(R12)
	ADDQ    $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_8x1_64Xor_end:
	RET

// func mulAvxTwo_8x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 73 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R12
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13
	ADDQ R14, R12

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_8x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (R11), Y9
	VMOVDQU 32(R11), Y11
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R13)
	VMOVDQU Y1, 32(R13)
	ADDQ    $0x40, R13
	VMOVDQU Y2, (R12)
	VMOVDQU Y3, 32(R12)
	ADDQ    $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x2_64_loop
	VZEROUPPER

mulAvxTwo_8x2_64_end:
	RET

// func mulGFNI_8x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), CX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R11
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12
	ADDQ R13, R11

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, CX

mulGFNI_8x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z18
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z18, Z16
	VGF2P8AFFINEQB $0x00, Z1, Z18, Z17

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z18
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z18
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z18
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z18
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z18
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (R10), Z18
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU64      (CX), Z18
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z15, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Store 2 outputs
	VMOVDQU64 Z16, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z17, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x2_64_loop
	VZEROUPPER

mulGFNI_8x2_64_end:
	RET

// func mulAvxGFNI_8x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x2(SB), $0-88
	// Loading 12 of 16 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R12
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13
	ADDQ R14, R12

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, DX

mulAvxGFNI_8x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x2_loop
	VZEROUPPER

mulAvxGFNI_8x2_end:
	RET

// func mulGFNI_8x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), CX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R11
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12
	ADDQ R13, R11

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, CX

mulGFNI_8x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (R12), Z16
	VMOVDQU64 (R11), Z17

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z18
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z18
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z18
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z18
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z18
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z18
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (R10), Z18
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU64      (CX), Z18
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
	VXORPD         Z16, Z19, Z16
	VGF2P8AFFINEQB $0x00, Z15, Z18, Z19
	VXORPD         Z17, Z19, Z17

	// Store 2 outputs
	VMOVDQU64 Z16, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z17, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x2_64Xor_loop
	VZEROUPPER

mulGFNI_8x2_64Xor_end:
	RET

// func mulAvxGFNI_8x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x2Xor(SB), $0-88
	// Loading 12 of 16 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 20 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R12
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13
	ADDQ R14, R12

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, DX

mulAvxGFNI_8x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (R13), Y12
	VMOVDQU (R12), Y13

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x2Xor_loop
	VZEROUPPER

mulAvxGFNI_8x2Xor_end:
	RET

// func mulAvxTwo_8x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 73 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R12
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13
	ADDQ R14, R12

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_8x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R13), Y0
	VMOVDQU 32(R13), Y1
	VMOVDQU (R12), Y2
	VMOVDQU 32(R12), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (R11), Y9
	VMOVDQU 32(R11), Y11
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R13)
	VMOVDQU Y1, 32(R13)
	ADDQ    $0x40, R13
	VMOVDQU Y2, (R12)
	VMOVDQU Y3, 32(R12)
	ADDQ    $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_8x2_64Xor_end:
	RET

// func mulAvxTwo_8x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x3_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 106 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R14
	MOVQ  48(R12), R12
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R12

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_8x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (R11), Y11
	VMOVDQU 32(R11), Y13
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R13)
	VMOVDQU Y1, 32(R13)
	ADDQ    $0x40, R13
	VMOVDQU Y2, (R14)
	VMOVDQU Y3, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y4, (R12)
	VMOVDQU Y5, 32(R12)
	ADDQ    $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x3_64_loop
	VZEROUPPER

mulAvxTwo_8x3_64_end:
	RET

// func mulGFNI_8x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), CX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R11
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R11

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, CX

mulGFNI_8x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z27
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z27, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z27, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z27, Z26

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z27
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z27
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z27
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z27
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z27
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (R10), Z27
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z18, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU64      (CX), Z27
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z21, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z22, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z23, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Store 3 outputs
	VMOVDQU64 Z24, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z25, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z26, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x3_64_loop
	VZEROUPPER

mulGFNI_8x3_64_end:
	RET

// func mulAvxGFNI_8x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x3(SB), $0-88
	// Loading 11 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R14
	MOVQ         48(R12), R12
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R12

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, DX

mulAvxGFNI_8x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x3_loop
	VZEROUPPER

mulAvxGFNI_8x3_end:
	RET

// func mulGFNI_8x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), CX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R11
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R12
	ADDQ R14, R13
	ADDQ R14, R11

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, CX

mulGFNI_8x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (R12), Z24
	VMOVDQU64 (R13), Z25
	VMOVDQU64 (R11), Z26

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z27
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z27
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z27
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z27
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z27
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z27
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (R10), Z27
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z18, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU64      (CX), Z27
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z21, Z27, Z28
	VXORPD         Z24, Z28, Z24
	VGF2P8AFFINEQB $0x00, Z22, Z27, Z28
	VXORPD         Z25, Z28, Z25
	VGF2P8AFFINEQB $0x00, Z23, Z27, Z28
	VXORPD         Z26, Z28, Z26

	// Store 3 outputs
	VMOVDQU64 Z24, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z25, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z26, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x3_64Xor_loop
	VZEROUPPER

mulGFNI_8x3_64Xor_end:
	RET

// func mulAvxGFNI_8x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x3Xor(SB), $0-88
	// Loading 11 of 24 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 29 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R14
	MOVQ         48(R12), R12
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R12

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, DX

mulAvxGFNI_8x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (R13), Y11
	VMOVDQU (R14), Y12
	VMOVDQU (R12), Y13

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x3Xor_loop
	VZEROUPPER

mulAvxGFNI_8x3Xor_end:
	RET

// func mulAvxTwo_8x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x3_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 106 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R14
	MOVQ  48(R12), R12
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R12

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_8x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R13), Y0
	VMOVDQU 32(R13), Y1
	VMOVDQU (R14), Y2
	VMOVDQU 32(R14), Y3
	VMOVDQU (R12), Y4
	VMOVDQU 32(R12), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (R11), Y11
	VMOVDQU 32(R11), Y13
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R13)
	VMOVDQU Y1, 32(R13)
	ADDQ    $0x40, R13
	VMOVDQU Y2, (R14)
	VMOVDQU Y3, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y4, (R12)
	VMOVDQU Y5, 32(R12)
	ADDQ    $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_8x3_64Xor_end:
	RET

// func mulAvxTwo_8x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x4(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 73 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R14
	MOVQ  48(R12), R15
	MOVQ  72(R12), R12
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_8x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (R11), Y7
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1792(CX), Y5
	VMOVDQU 1824(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1856(CX), Y5
	VMOVDQU 1888(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1920(CX), Y5
	VMOVDQU 1952(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1984(CX), Y5
	VMOVDQU 2016(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y1, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y2, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x4_loop
	VZEROUPPER

mulAvxTwo_8x4_end:
	RET

// func mulGFNI_8x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x4_64(SB), $8-88
	// Loading 26 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R14
	MOVQ            48(R12), R15
	MOVQ            72(R12), R12
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, DX

mulGFNI_8x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 4 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x4_64_loop
	VZEROUPPER

mulGFNI_8x4_64_end:
	RET

// func mulAvxGFNI_8x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x4(SB), $8-88
	// Loading 10 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R14
	MOVQ         48(R12), R15
	MOVQ         72(R12), R12
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, DX

mulAvxGFNI_8x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x4_loop
	VZEROUPPER

mulAvxGFNI_8x4_end:
	RET

// func mulGFNI_8x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x4_64Xor(SB), $8-88
	// Loading 26 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R14
	MOVQ            48(R12), R15
	MOVQ            72(R12), R12
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, DX

mulGFNI_8x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R12), Z29

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 4 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_8x4_64Xor_loop
	VZEROUPPER

mulGFNI_8x4_64Xor_end:
	RET

// func mulAvxGFNI_8x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x4Xor(SB), $8-88
	// Loading 10 of 32 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 38 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R14
	MOVQ         48(R12), R15
	MOVQ         72(R12), R12
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, DX

mulAvxGFNI_8x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R12), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_8x4Xor_loop
	VZEROUPPER

mulAvxGFNI_8x4Xor_end:
	RET

// func mulAvxTwo_8x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x4Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 73 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R14
	MOVQ  48(R12), R15
	MOVQ  72(R12), R12
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_8x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (R13), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R14), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R15), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (R12), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (R11), Y7
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1792(CX), Y5
	VMOVDQU 1824(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1856(CX), Y5
	VMOVDQU 1888(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1920(CX), Y5
	VMOVDQU 1952(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1984(CX), Y5
	VMOVDQU 2016(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y1, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y2, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_8x4Xor_loop
	VZEROUPPER

mulAvxTwo_8x4Xor_end:
	RET

// func mulAvxTwo_8x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x5(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 90 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x5_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), R10
	MOVQ  168(AX), AX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R14
	MOVQ  72(R11), R15
	MOVQ  96(R11), R11
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X5
	VPBROADCASTB X5, Y5
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_8x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU (AX), Y8
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2240(CX), Y6
	VMOVDQU 2272(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2304(CX), Y6
	VMOVDQU 2336(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2368(CX), Y6
	VMOVDQU 2400(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2432(CX), Y6
	VMOVDQU 2464(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2496(CX), Y6
	VMOVDQU 2528(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y1, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y2, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y3, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_8x5_loop
	VZEROUPPER

mulAvxTwo_8x5_end:
	RET

// func mulGFNI_8x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x5_64(SB), $8-88
	// Loading 25 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), R10
	MOVQ            168(AX), AX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R14
	MOVQ            72(R11), R15
	MOVQ            96(R11), R11
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_8x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 5 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_8x5_64_loop
	VZEROUPPER

mulGFNI_8x5_64_end:
	RET

// func mulAvxGFNI_8x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x5(SB), $8-88
	// Loading 9 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), R10
	MOVQ         168(AX), AX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R14
	MOVQ         72(R11), R15
	MOVQ         96(R11), R11
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_8x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_8x5_loop
	VZEROUPPER

mulAvxGFNI_8x5_end:
	RET

// func mulGFNI_8x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x5_64Xor(SB), $8-88
	// Loading 25 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), R10
	MOVQ            168(AX), AX
	MOVQ            out_base+48(FP), R11
	MOVQ            out_base+48(FP), R11
	MOVQ            (R11), R12
	MOVQ            24(R11), R13
	MOVQ            48(R11), R14
	MOVQ            72(R11), R15
	MOVQ            96(R11), R11
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_8x5_64Xor_loop:
	// Load 5 outputs
	VMOVDQU64 (R12), Z25
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R11), Z29

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 5 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	VMOVDQU64 Z25, (R12)
	ADDQ      $0x40, R12
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R11)
	ADDQ      $0x40, R11

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_8x5_64Xor_loop
	VZEROUPPER

mulGFNI_8x5_64Xor_end:
	RET

// func mulAvxGFNI_8x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x5Xor(SB), $8-88
	// Loading 9 of 40 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 47 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), R10
	MOVQ         168(AX), AX
	MOVQ         out_base+48(FP), R11
	MOVQ         out_base+48(FP), R11
	MOVQ         (R11), R12
	MOVQ         24(R11), R13
	MOVQ         48(R11), R14
	MOVQ         72(R11), R15
	MOVQ         96(R11), R11
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_8x5Xor_loop:
	// Load 5 outputs
	VMOVDQU (R12), Y9
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R11), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_8x5Xor_loop
	VZEROUPPER

mulAvxGFNI_8x5Xor_end:
	RET

// func mulAvxTwo_8x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x5Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 90 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x5Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), R10
	MOVQ  168(AX), AX
	MOVQ  out_base+48(FP), R11
	MOVQ  (R11), R12
	MOVQ  24(R11), R13
	MOVQ  48(R11), R14
	MOVQ  72(R11), R15
	MOVQ  96(R11), R11
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R11

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X5
	VPBROADCASTB X5, Y5
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_8x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (R12), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU (R13), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU (R14), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU (R15), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU (R11), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU (AX), Y8
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2240(CX), Y6
	VMOVDQU 2272(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2304(CX), Y6
	VMOVDQU 2336(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2368(CX), Y6
	VMOVDQU 2400(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2432(CX), Y6
	VMOVDQU 2464(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2496(CX), Y6
	VMOVDQU 2528(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	VMOVDQU Y0, (R12)
	ADDQ    $0x20, R12
	VMOVDQU Y1, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y2, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y3, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y4, (R11)
	ADDQ    $0x20, R11

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_8x5Xor_loop
	VZEROUPPER

mulAvxTwo_8x5Xor_end:
	RET

// func mulAvxTwo_8x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x6(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 107 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_8x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R10), Y9
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (R11), Y9
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2688(CX), Y7
	VMOVDQU 2720(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2752(CX), Y7
	VMOVDQU 2784(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2816(CX), Y7
	VMOVDQU 2848(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2880(CX), Y7
	VMOVDQU 2912(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2944(CX), Y7
	VMOVDQU 2976(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3008(CX), Y7
	VMOVDQU 3040(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x6_loop
	VZEROUPPER

mulAvxTwo_8x6_end:
	RET

// func mulGFNI_8x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x6_64(SB), $0-88
	// Loading 24 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x6_64_loop
	VZEROUPPER

mulGFNI_8x6_64_end:
	RET

// func mulAvxGFNI_8x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x6(SB), $0-88
	// Loading 8 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	MOVQ    (R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x6_loop
	VZEROUPPER

mulAvxGFNI_8x6_end:
	RET

// func mulGFNI_8x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x6_64Xor(SB), $0-88
	// Loading 24 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x6_64Xor_loop:
	// Load 6 outputs
	MOVQ      (R12), R14
	VMOVDQU64 (R14)(R13*1), Z24
	MOVQ      24(R12), R14
	VMOVDQU64 (R14)(R13*1), Z25
	MOVQ      48(R12), R14
	VMOVDQU64 (R14)(R13*1), Z26
	MOVQ      72(R12), R14
	VMOVDQU64 (R14)(R13*1), Z27
	MOVQ      96(R12), R14
	VMOVDQU64 (R14)(R13*1), Z28
	MOVQ      120(R12), R14
	VMOVDQU64 (R14)(R13*1), Z29

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x6_64Xor_loop
	VZEROUPPER

mulGFNI_8x6_64Xor_end:
	RET

// func mulAvxGFNI_8x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x6Xor(SB), $0-88
	// Loading 8 of 48 tables to registers
	// Destination kept on stack
	// Full registers estimated 56 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x6Xor_loop:
	// Load 6 outputs
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y8
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y9
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y10
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y11
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y12
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	MOVQ    (R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x6Xor_loop
	VZEROUPPER

mulAvxGFNI_8x6Xor_end:
	RET

// func mulAvxTwo_8x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x6Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 107 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_8x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R10), Y9
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (R11), Y9
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2688(CX), Y7
	VMOVDQU 2720(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2752(CX), Y7
	VMOVDQU 2784(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2816(CX), Y7
	VMOVDQU 2848(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2880(CX), Y7
	VMOVDQU 2912(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2944(CX), Y7
	VMOVDQU 2976(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3008(CX), Y7
	VMOVDQU 3040(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x6Xor_loop
	VZEROUPPER

mulAvxTwo_8x6Xor_end:
	RET

// func mulAvxTwo_8x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x7(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 124 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_8x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (R11), Y10
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3136(CX), Y8
	VMOVDQU 3168(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3200(CX), Y8
	VMOVDQU 3232(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3264(CX), Y8
	VMOVDQU 3296(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3328(CX), Y8
	VMOVDQU 3360(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3392(CX), Y8
	VMOVDQU 3424(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3456(CX), Y8
	VMOVDQU 3488(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3520(CX), Y8
	VMOVDQU 3552(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x7_loop
	VZEROUPPER

mulAvxTwo_8x7_end:
	RET

// func mulGFNI_8x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x7_64(SB), $0-88
	// Loading 23 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x7_64_loop
	VZEROUPPER

mulGFNI_8x7_64_end:
	RET

// func mulAvxGFNI_8x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x7(SB), $0-88
	// Loading 7 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x7_loop
	VZEROUPPER

mulAvxGFNI_8x7_end:
	RET

// func mulGFNI_8x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x7_64Xor(SB), $0-88
	// Loading 23 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x7_64Xor_loop:
	// Load 7 outputs
	MOVQ      (R12), R14
	VMOVDQU64 (R14)(R13*1), Z23
	MOVQ      24(R12), R14
	VMOVDQU64 (R14)(R13*1), Z24
	MOVQ      48(R12), R14
	VMOVDQU64 (R14)(R13*1), Z25
	MOVQ      72(R12), R14
	VMOVDQU64 (R14)(R13*1), Z26
	MOVQ      96(R12), R14
	VMOVDQU64 (R14)(R13*1), Z27
	MOVQ      120(R12), R14
	VMOVDQU64 (R14)(R13*1), Z28
	MOVQ      144(R12), R14
	VMOVDQU64 (R14)(R13*1), Z29

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x7_64Xor_loop
	VZEROUPPER

mulGFNI_8x7_64Xor_end:
	RET

// func mulAvxGFNI_8x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x7Xor(SB), $0-88
	// Loading 7 of 56 tables to registers
	// Destination kept on stack
	// Full registers estimated 65 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x7Xor_loop:
	// Load 7 outputs
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y7
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y8
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y9
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y10
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y11
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y12
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x7Xor_loop
	VZEROUPPER

mulAvxGFNI_8x7Xor_end:
	RET

// func mulAvxTwo_8x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x7Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 124 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_8x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (R11), Y10
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3136(CX), Y8
	VMOVDQU 3168(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3200(CX), Y8
	VMOVDQU 3232(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3264(CX), Y8
	VMOVDQU 3296(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3328(CX), Y8
	VMOVDQU 3360(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3392(CX), Y8
	VMOVDQU 3424(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3456(CX), Y8
	VMOVDQU 3488(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3520(CX), Y8
	VMOVDQU 3552(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x7Xor_loop
	VZEROUPPER

mulAvxTwo_8x7Xor_end:
	RET

// func mulAvxTwo_8x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x8(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 141 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_8x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (R11), Y11
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3584(CX), Y9
	VMOVDQU 3616(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3648(CX), Y9
	VMOVDQU 3680(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3712(CX), Y9
	VMOVDQU 3744(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3776(CX), Y9
	VMOVDQU 3808(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3840(CX), Y9
	VMOVDQU 3872(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3904(CX), Y9
	VMOVDQU 3936(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3968(CX), Y9
	VMOVDQU 4000(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4032(CX), Y9
	VMOVDQU 4064(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y7, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x8_loop
	VZEROUPPER

mulAvxTwo_8x8_end:
	RET

// func mulGFNI_8x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x8_64(SB), $0-88
	// Loading 22 of 64 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z22, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      168(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x8_64_loop
	VZEROUPPER

mulGFNI_8x8_64_end:
	RET

// func mulAvxGFNI_8x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x8(SB), $0-88
	// Loading 6 of 64 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x8_loop
	VZEROUPPER

mulAvxGFNI_8x8_end:
	RET

// func mulGFNI_8x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x8_64Xor(SB), $0-88
	// Loading 22 of 64 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x8_64Xor_loop:
	// Load 8 outputs
	MOVQ      (R12), R14
	VMOVDQU64 (R14)(R13*1), Z22
	MOVQ      24(R12), R14
	VMOVDQU64 (R14)(R13*1), Z23
	MOVQ      48(R12), R14
	VMOVDQU64 (R14)(R13*1), Z24
	MOVQ      72(R12), R14
	VMOVDQU64 (R14)(R13*1), Z25
	MOVQ      96(R12), R14
	VMOVDQU64 (R14)(R13*1), Z26
	MOVQ      120(R12), R14
	VMOVDQU64 (R14)(R13*1), Z27
	MOVQ      144(R12), R14
	VMOVDQU64 (R14)(R13*1), Z28
	MOVQ      168(R12), R14
	VMOVDQU64 (R14)(R13*1), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z22, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      168(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x8_64Xor_loop
	VZEROUPPER

mulGFNI_8x8_64Xor_end:
	RET

// func mulAvxGFNI_8x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x8Xor(SB), $0-88
	// Loading 6 of 64 tables to registers
	// Destination kept on stack
	// Full registers estimated 74 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x8Xor_loop:
	// Load 8 outputs
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y6
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y7
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y8
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y9
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y10
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y11
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y12
	MOVQ    168(R12), R14
	VMOVDQU (R14)(R13*1), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x8Xor_loop
	VZEROUPPER

mulAvxGFNI_8x8Xor_end:
	RET

// func mulAvxTwo_8x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x8Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 141 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_8x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	MOVQ    168(R12), R14
	VMOVDQU (R14)(R13*1), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (R11), Y11
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3584(CX), Y9
	VMOVDQU 3616(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3648(CX), Y9
	VMOVDQU 3680(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3712(CX), Y9
	VMOVDQU 3744(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3776(CX), Y9
	VMOVDQU 3808(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3840(CX), Y9
	VMOVDQU 3872(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3904(CX), Y9
	VMOVDQU 3936(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3968(CX), Y9
	VMOVDQU 4000(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4032(CX), Y9
	VMOVDQU 4064(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y7, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x8Xor_loop
	VZEROUPPER

mulAvxTwo_8x8Xor_end:
	RET

// func mulAvxTwo_8x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x9(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 158 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_8x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (R11), Y12
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4032(CX), Y10
	VMOVDQU 4064(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4096(CX), Y10
	VMOVDQU 4128(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4160(CX), Y10
	VMOVDQU 4192(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4224(CX), Y10
	VMOVDQU 4256(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4288(CX), Y10
	VMOVDQU 4320(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4352(CX), Y10
	VMOVDQU 4384(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4416(CX), Y10
	VMOVDQU 4448(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 4480(CX), Y10
	VMOVDQU 4512(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 4544(CX), Y10
	VMOVDQU 4576(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y8, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x9_loop
	VZEROUPPER

mulAvxTwo_8x9_end:
	RET

// func mulGFNI_8x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x9_64(SB), $0-88
	// Loading 21 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 83 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z21, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z22, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      168(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      192(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x9_64_loop
	VZEROUPPER

mulGFNI_8x9_64_end:
	RET

// func mulAvxGFNI_8x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x9(SB), $0-88
	// Loading 5 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 83 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x9_loop
	VZEROUPPER

mulAvxGFNI_8x9_end:
	RET

// func mulGFNI_8x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x9_64Xor(SB), $0-88
	// Loading 21 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 83 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x9_64Xor_loop:
	// Load 9 outputs
	MOVQ      (R12), R14
	VMOVDQU64 (R14)(R13*1), Z21
	MOVQ      24(R12), R14
	VMOVDQU64 (R14)(R13*1), Z22
	MOVQ      48(R12), R14
	VMOVDQU64 (R14)(R13*1), Z23
	MOVQ      72(R12), R14
	VMOVDQU64 (R14)(R13*1), Z24
	MOVQ      96(R12), R14
	VMOVDQU64 (R14)(R13*1), Z25
	MOVQ      120(R12), R14
	VMOVDQU64 (R14)(R13*1), Z26
	MOVQ      144(R12), R14
	VMOVDQU64 (R14)(R13*1), Z27
	MOVQ      168(R12), R14
	VMOVDQU64 (R14)(R13*1), Z28
	MOVQ      192(R12), R14
	VMOVDQU64 (R14)(R13*1), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z21, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z22, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      168(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      192(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x9_64Xor_loop
	VZEROUPPER

mulGFNI_8x9_64Xor_end:
	RET

// func mulAvxGFNI_8x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x9Xor(SB), $0-88
	// Loading 5 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 83 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x9Xor_loop:
	// Load 9 outputs
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y5
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y6
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y7
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y8
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y9
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y10
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y11
	MOVQ    168(R12), R14
	VMOVDQU (R14)(R13*1), Y12
	MOVQ    192(R12), R14
	VMOVDQU (R14)(R13*1), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x9Xor_loop
	VZEROUPPER

mulAvxGFNI_8x9Xor_end:
	RET

// func mulAvxTwo_8x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x9Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 158 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_8x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	MOVQ    168(R12), R14
	VMOVDQU (R14)(R13*1), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	MOVQ    192(R12), R14
	VMOVDQU (R14)(R13*1), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (R11), Y12
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4032(CX), Y10
	VMOVDQU 4064(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4096(CX), Y10
	VMOVDQU 4128(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4160(CX), Y10
	VMOVDQU 4192(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4224(CX), Y10
	VMOVDQU 4256(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4288(CX), Y10
	VMOVDQU 4320(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4352(CX), Y10
	VMOVDQU 4384(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4416(CX), Y10
	VMOVDQU 4448(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 4480(CX), Y10
	VMOVDQU 4512(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 4544(CX), Y10
	VMOVDQU 4576(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y8, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x9Xor_loop
	VZEROUPPER

mulAvxTwo_8x9Xor_end:
	RET

// func mulAvxTwo_8x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x10(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 175 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_8x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (R11), Y13
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 4480(CX), Y11
	VMOVDQU 4512(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 4544(CX), Y11
	VMOVDQU 4576(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 4608(CX), Y11
	VMOVDQU 4640(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4672(CX), Y11
	VMOVDQU 4704(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4736(CX), Y11
	VMOVDQU 4768(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4800(CX), Y11
	VMOVDQU 4832(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4864(CX), Y11
	VMOVDQU 4896(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4928(CX), Y11
	VMOVDQU 4960(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4992(CX), Y11
	VMOVDQU 5024(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5056(CX), Y11
	VMOVDQU 5088(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    216(R12), R14
	VMOVDQU Y9, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x10_loop
	VZEROUPPER

mulAvxTwo_8x10_end:
	RET

// func mulGFNI_8x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x10_64(SB), $0-88
	// Loading 20 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z20, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z21, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z22, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      168(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      192(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      216(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x10_64_loop
	VZEROUPPER

mulGFNI_8x10_64_end:
	RET

// func mulAvxGFNI_8x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x10(SB), $0-88
	// Loading 4 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    216(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x10_loop
	VZEROUPPER

mulAvxGFNI_8x10_end:
	RET

// func mulGFNI_8x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_8x10_64Xor(SB), $0-88
	// Loading 20 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_8x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), DX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulGFNI_8x10_64Xor_loop:
	// Load 10 outputs
	MOVQ      (R12), R14
	VMOVDQU64 (R14)(R13*1), Z20
	MOVQ      24(R12), R14
	VMOVDQU64 (R14)(R13*1), Z21
	MOVQ      48(R12), R14
	VMOVDQU64 (R14)(R13*1), Z22
	MOVQ      72(R12), R14
	VMOVDQU64 (R14)(R13*1), Z23
	MOVQ      96(R12), R14
	VMOVDQU64 (R14)(R13*1), Z24
	MOVQ      120(R12), R14
	VMOVDQU64 (R14)(R13*1), Z25
	MOVQ      144(R12), R14
	VMOVDQU64 (R14)(R13*1), Z26
	MOVQ      168(R12), R14
	VMOVDQU64 (R14)(R13*1), Z27
	MOVQ      192(R12), R14
	VMOVDQU64 (R14)(R13*1), Z28
	MOVQ      216(R12), R14
	VMOVDQU64 (R14)(R13*1), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R12), R14
	VMOVDQU64 Z20, (R14)(R13*1)
	MOVQ      24(R12), R14
	VMOVDQU64 Z21, (R14)(R13*1)
	MOVQ      48(R12), R14
	VMOVDQU64 Z22, (R14)(R13*1)
	MOVQ      72(R12), R14
	VMOVDQU64 Z23, (R14)(R13*1)
	MOVQ      96(R12), R14
	VMOVDQU64 Z24, (R14)(R13*1)
	MOVQ      120(R12), R14
	VMOVDQU64 Z25, (R14)(R13*1)
	MOVQ      144(R12), R14
	VMOVDQU64 Z26, (R14)(R13*1)
	MOVQ      168(R12), R14
	VMOVDQU64 Z27, (R14)(R13*1)
	MOVQ      192(R12), R14
	VMOVDQU64 Z28, (R14)(R13*1)
	MOVQ      216(R12), R14
	VMOVDQU64 Z29, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x40, R13
	DECQ AX
	JNZ  mulGFNI_8x10_64Xor_loop
	VZEROUPPER

mulGFNI_8x10_64Xor_end:
	RET

// func mulAvxGFNI_8x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_8x10Xor(SB), $0-88
	// Loading 4 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_8x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), DX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         start+72(FP), R13

	// Add start offset to input
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, DX

mulAvxGFNI_8x10Xor_loop:
	// Load 10 outputs
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y4
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y5
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y6
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y7
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y8
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y9
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y10
	MOVQ    168(R12), R14
	VMOVDQU (R14)(R13*1), Y11
	MOVQ    192(R12), R14
	VMOVDQU (R14)(R13*1), Y12
	MOVQ    216(R12), R14
	VMOVDQU (R14)(R13*1), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y9, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y10, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y11, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y12, (R14)(R13*1)
	MOVQ    216(R12), R14
	VMOVDQU Y13, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxGFNI_8x10Xor_loop
	VZEROUPPER

mulAvxGFNI_8x10Xor_end:
	RET

// func mulAvxTwo_8x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_8x10Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 175 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_8x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), DX
	MOVQ  out_base+48(FP), R12
	MOVQ  start+72(FP), R13

	// Add start offset to input
	ADDQ         R13, BX
	ADDQ         R13, SI
	ADDQ         R13, DI
	ADDQ         R13, R8
	ADDQ         R13, R9
	ADDQ         R13, R10
	ADDQ         R13, R11
	ADDQ         R13, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_8x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	MOVQ    (R12), R14
	VMOVDQU (R14)(R13*1), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	MOVQ    24(R12), R14
	VMOVDQU (R14)(R13*1), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	MOVQ    48(R12), R14
	VMOVDQU (R14)(R13*1), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	MOVQ    72(R12), R14
	VMOVDQU (R14)(R13*1), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	MOVQ    96(R12), R14
	VMOVDQU (R14)(R13*1), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	MOVQ    120(R12), R14
	VMOVDQU (R14)(R13*1), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	MOVQ    144(R12), R14
	VMOVDQU (R14)(R13*1), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	MOVQ    168(R12), R14
	VMOVDQU (R14)(R13*1), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	MOVQ    192(R12), R14
	VMOVDQU (R14)(R13*1), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	MOVQ    216(R12), R14
	VMOVDQU (R14)(R13*1), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (R11), Y13
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 4480(CX), Y11
	VMOVDQU 4512(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 4544(CX), Y11
	VMOVDQU 4576(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 4608(CX), Y11
	VMOVDQU 4640(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4672(CX), Y11
	VMOVDQU 4704(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4736(CX), Y11
	VMOVDQU 4768(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4800(CX), Y11
	VMOVDQU 4832(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4864(CX), Y11
	VMOVDQU 4896(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4928(CX), Y11
	VMOVDQU 4960(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4992(CX), Y11
	VMOVDQU 5024(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5056(CX), Y11
	VMOVDQU 5088(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R12), R14
	VMOVDQU Y0, (R14)(R13*1)
	MOVQ    24(R12), R14
	VMOVDQU Y1, (R14)(R13*1)
	MOVQ    48(R12), R14
	VMOVDQU Y2, (R14)(R13*1)
	MOVQ    72(R12), R14
	VMOVDQU Y3, (R14)(R13*1)
	MOVQ    96(R12), R14
	VMOVDQU Y4, (R14)(R13*1)
	MOVQ    120(R12), R14
	VMOVDQU Y5, (R14)(R13*1)
	MOVQ    144(R12), R14
	VMOVDQU Y6, (R14)(R13*1)
	MOVQ    168(R12), R14
	VMOVDQU Y7, (R14)(R13*1)
	MOVQ    192(R12), R14
	VMOVDQU Y8, (R14)(R13*1)
	MOVQ    216(R12), R14
	VMOVDQU Y9, (R14)(R13*1)

	// Prepare for next loop
	ADDQ $0x20, R13
	DECQ AX
	JNZ  mulAvxTwo_8x10Xor_loop
	VZEROUPPER

mulAvxTwo_8x10Xor_end:
	RET

// func mulAvxTwo_9x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R13
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_9x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (R11), Y6
	VMOVDQU 32(R11), Y5
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU (R12), Y6
	VMOVDQU 32(R12), Y5
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 448(CX), Y3
	VMOVDQU 480(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 512(CX), Y3
	VMOVDQU 544(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R13)
	VMOVDQU Y1, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_9x1_64_loop
	VZEROUPPER

mulAvxTwo_9x1_64_end:
	RET

// func mulGFNI_9x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), CX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R12
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, CX

mulGFNI_9x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z10
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z10, Z9

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z10
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z10
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z10
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z10
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z10
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (R10), Z10
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z6, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU64      (R11), Z10
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z7, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU64      (CX), Z10
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Store 1 outputs
	VMOVDQU64 Z9, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_9x1_64_loop
	VZEROUPPER

mulGFNI_9x1_64_end:
	RET

// func mulAvxGFNI_9x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), R10
	MOVQ         168(CX), R11
	MOVQ         192(CX), CX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R12
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, CX

mulAvxGFNI_9x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y10
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y10, Y9

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y10
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y10
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y10
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y10
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y10
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (R10), Y10
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y6, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 7 to 1 outputs
	VMOVDQU        (R11), Y10
	ADDQ           $0x20, R11
	VGF2P8AFFINEQB $0x00, Y7, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 8 to 1 outputs
	VMOVDQU        (CX), Y10
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y8, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Store 1 outputs
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_9x1_loop
	VZEROUPPER

mulAvxGFNI_9x1_end:
	RET

// func mulGFNI_9x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), CX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R12
	MOVQ            start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, CX

mulGFNI_9x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (R12), Z9

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z10
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z10
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z10
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z10
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z10
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z10
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (R10), Z10
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z6, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU64      (R11), Z10
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z7, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU64      (CX), Z10
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z8, Z10, Z10
	VXORPD         Z9, Z10, Z9

	// Store 1 outputs
	VMOVDQU64 Z9, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_9x1_64Xor_loop
	VZEROUPPER

mulGFNI_9x1_64Xor_end:
	RET

// func mulAvxGFNI_9x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 12 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), R10
	MOVQ         168(CX), R11
	MOVQ         192(CX), CX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R12
	MOVQ         start+72(FP), R13

	// Add start offset to output
	ADDQ R13, R12

	// Add start offset to input
	ADDQ R13, DX
	ADDQ R13, BX
	ADDQ R13, SI
	ADDQ R13, DI
	ADDQ R13, R8
	ADDQ R13, R9
	ADDQ R13, R10
	ADDQ R13, R11
	ADDQ R13, CX

mulAvxGFNI_9x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (R12), Y9

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y10
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y10
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y10
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y10
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y10
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y10
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (R10), Y10
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y6, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 7 to 1 outputs
	VMOVDQU        (R11), Y10
	ADDQ           $0x20, R11
	VGF2P8AFFINEQB $0x00, Y7, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Load and process 32 bytes from input 8 to 1 outputs
	VMOVDQU        (CX), Y10
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y8, Y10, Y10
	VXORPD         Y9, Y10, Y9

	// Store 1 outputs
	VMOVDQU Y9, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_9x1Xor_loop
	VZEROUPPER

mulAvxGFNI_9x1Xor_end:
	RET

// func mulAvxTwo_9x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R13
	MOVQ  start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R14
	MOVQ         R14, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_9x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (R13), Y0
	VMOVDQU 32(R13), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (R11), Y6
	VMOVDQU 32(R11), Y5
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU (R12), Y6
	VMOVDQU 32(R12), Y5
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 448(CX), Y3
	VMOVDQU 480(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 512(CX), Y3
	VMOVDQU 544(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R13)
	VMOVDQU Y1, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_9x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_9x1_64Xor_end:
	RET

// func mulAvxTwo_9x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x2_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 81 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R14
	MOVQ  24(R13), R13
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14
	ADDQ R15, R13

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_9x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (R11), Y9
	VMOVDQU 32(R11), Y11
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU (R12), Y9
	VMOVDQU 32(R12), Y11
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y2, (R13)
	VMOVDQU Y3, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_9x2_64_loop
	VZEROUPPER

mulAvxTwo_9x2_64_end:
	RET

// func mulGFNI_9x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), CX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R12
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13
	ADDQ R14, R12

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, CX

mulGFNI_9x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z20
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z20, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z20, Z19

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z20
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z20
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z20
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z20
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z20
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (R10), Z20
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU64      (R11), Z20
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU64      (CX), Z20
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z16, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z17, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Store 2 outputs
	VMOVDQU64 Z18, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z19, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_9x2_64_loop
	VZEROUPPER

mulGFNI_9x2_64_end:
	RET

// func mulAvxGFNI_9x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x2(SB), $0-88
	// Loading 12 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R14
	MOVQ         24(R13), R13
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14
	ADDQ R15, R13

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, DX

mulAvxGFNI_9x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 2 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_9x2_loop
	VZEROUPPER

mulAvxGFNI_9x2_end:
	RET

// func mulGFNI_9x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), CX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R12
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13
	ADDQ R14, R12

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, CX

mulGFNI_9x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (R13), Z18
	VMOVDQU64 (R12), Z19

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z20
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z20
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z20
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z20
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z20
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z20
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (R10), Z20
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU64      (R11), Z20
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU64      (CX), Z20
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z16, Z20, Z21
	VXORPD         Z18, Z21, Z18
	VGF2P8AFFINEQB $0x00, Z17, Z20, Z21
	VXORPD         Z19, Z21, Z19

	// Store 2 outputs
	VMOVDQU64 Z18, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z19, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_9x2_64Xor_loop
	VZEROUPPER

mulGFNI_9x2_64Xor_end:
	RET

// func mulAvxGFNI_9x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x2Xor(SB), $0-88
	// Loading 12 of 18 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 22 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R14
	MOVQ         24(R13), R13
	MOVQ         start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14
	ADDQ R15, R13

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, DX

mulAvxGFNI_9x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (R14), Y12
	VMOVDQU (R13), Y13

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 2 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y13, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_9x2Xor_loop
	VZEROUPPER

mulAvxGFNI_9x2Xor_end:
	RET

// func mulAvxTwo_9x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x2_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 81 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R14
	MOVQ  24(R13), R13
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14
	ADDQ R15, R13

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_9x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R14), Y0
	VMOVDQU 32(R14), Y1
	VMOVDQU (R13), Y2
	VMOVDQU 32(R13), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (R11), Y9
	VMOVDQU 32(R11), Y11
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU (R12), Y9
	VMOVDQU 32(R12), Y11
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y2, (R13)
	VMOVDQU Y3, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_9x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_9x2_64Xor_end:
	RET

// func mulAvxTwo_9x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x3_64(SB), $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 118 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x3_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R14
	MOVQ  24(R13), R15
	MOVQ  48(R13), R13
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, R12
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_9x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (R11), Y11
	VMOVDQU 32(R11), Y13
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU (R12), Y11
	VMOVDQU 32(R12), Y13
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y2, (R15)
	VMOVDQU Y3, 32(R15)
	ADDQ    $0x40, R15
	VMOVDQU Y4, (R13)
	VMOVDQU Y5, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_9x3_64_loop
	VZEROUPPER

mulAvxTwo_9x3_64_end:
	RET

// func mulGFNI_9x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x3_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	VBROADCASTF32X2 208(CX), Z26
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), CX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R14
	MOVQ            48(R12), R12
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R12

	// Add start offset to input
	ADDQ R15, DX
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, CX

mulGFNI_9x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z29

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU64      (R11), Z30
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 3 outputs
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_9x3_64_loop
	VZEROUPPER

mulGFNI_9x3_64_end:
	RET

// func mulAvxGFNI_9x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x3(SB), $8-88
	// Loading 11 of 27 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R14
	MOVQ         24(R13), R15
	MOVQ         48(R13), R13
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, DX

mulAvxGFNI_9x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 3 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_9x3_loop
	VZEROUPPER

mulAvxGFNI_9x3_end:
	RET

// func mulGFNI_9x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x3_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	VBROADCASTF32X2 208(CX), Z26
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), CX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R14
	MOVQ            48(R12), R12
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R13
	ADDQ R15, R14
	ADDQ R15, R12

	// Add start offset to input
	ADDQ R15, DX
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, CX

mulGFNI_9x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (R13), Z27
	VMOVDQU64 (R14), Z28
	VMOVDQU64 (R12), Z29

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU64      (R11), Z30
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU64      (CX), Z30
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Store 3 outputs
	VMOVDQU64 Z27, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z28, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z29, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_9x3_64Xor_loop
	VZEROUPPER

mulGFNI_9x3_64Xor_end:
	RET

// func mulAvxGFNI_9x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x3Xor(SB), $8-88
	// Loading 11 of 27 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 32 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R14
	MOVQ         24(R13), R15
	MOVQ         48(R13), R13
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, DX

mulAvxGFNI_9x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R13), Y13

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 3 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_9x3Xor_loop
	VZEROUPPER

mulAvxGFNI_9x3Xor_end:
	RET

// func mulAvxTwo_9x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x3_64Xor(SB), $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 118 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x3_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R14
	MOVQ  24(R13), R15
	MOVQ  48(R13), R13
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, R12
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_9x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R14), Y0
	VMOVDQU 32(R14), Y1
	VMOVDQU (R15), Y2
	VMOVDQU 32(R15), Y3
	VMOVDQU (R13), Y4
	VMOVDQU 32(R13), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (R11), Y11
	VMOVDQU 32(R11), Y13
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU (R12), Y11
	VMOVDQU 32(R12), Y13
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y2, (R15)
	VMOVDQU Y3, 32(R15)
	ADDQ    $0x40, R15
	VMOVDQU Y4, (R13)
	VMOVDQU Y5, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_9x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_9x3_64Xor_end:
	RET

// func mulAvxTwo_9x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x4(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 81 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x4_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), R10
	MOVQ  168(AX), R11
	MOVQ  192(AX), AX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R14
	MOVQ  48(R12), R15
	MOVQ  72(R12), R12
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_9x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU (R11), Y7
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1792(CX), Y5
	VMOVDQU 1824(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1856(CX), Y5
	VMOVDQU 1888(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1920(CX), Y5
	VMOVDQU 1952(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1984(CX), Y5
	VMOVDQU 2016(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU (AX), Y7
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 2048(CX), Y5
	VMOVDQU 2080(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 2112(CX), Y5
	VMOVDQU 2144(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 2176(CX), Y5
	VMOVDQU 2208(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 2240(CX), Y5
	VMOVDQU 2272(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y1, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y2, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_9x4_loop
	VZEROUPPER

mulAvxTwo_9x4_end:
	RET

// func mulGFNI_9x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x4_64(SB), $8-88
	// Loading 26 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), R10
	MOVQ            168(AX), R11
	MOVQ            192(AX), AX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R14
	MOVQ            48(R12), R15
	MOVQ            72(R12), R12
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_9x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 4 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 4 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_9x4_64_loop
	VZEROUPPER

mulGFNI_9x4_64_end:
	RET

// func mulAvxGFNI_9x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x4(SB), $8-88
	// Loading 10 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), R10
	MOVQ         168(AX), R11
	MOVQ         192(AX), AX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R14
	MOVQ         48(R12), R15
	MOVQ         72(R12), R12
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_9x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_9x4_loop
	VZEROUPPER

mulAvxGFNI_9x4_end:
	RET

// func mulGFNI_9x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x4_64Xor(SB), $8-88
	// Loading 26 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), R10
	MOVQ            168(AX), R11
	MOVQ            192(AX), AX
	MOVQ            out_base+48(FP), R12
	MOVQ            out_base+48(FP), R12
	MOVQ            (R12), R13
	MOVQ            24(R12), R14
	MOVQ            48(R12), R15
	MOVQ            72(R12), R12
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_9x4_64Xor_loop:
	// Load 4 outputs
	VMOVDQU64 (R13), Z26
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R12), Z29

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 4 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 4 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	VMOVDQU64 Z26, (R13)
	ADDQ      $0x40, R13
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R12)
	ADDQ      $0x40, R12

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_9x4_64Xor_loop
	VZEROUPPER

mulGFNI_9x4_64Xor_end:
	RET

// func mulAvxGFNI_9x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x4Xor(SB), $8-88
	// Loading 10 of 36 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 42 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), R10
	MOVQ         168(AX), R11
	MOVQ         192(AX), AX
	MOVQ         out_base+48(FP), R12
	MOVQ         out_base+48(FP), R12
	MOVQ         (R12), R13
	MOVQ         24(R12), R14
	MOVQ         48(R12), R15
	MOVQ         72(R12), R12
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_9x4Xor_loop:
	// Load 4 outputs
	VMOVDQU (R13), Y10
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R12), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_9x4Xor_loop
	VZEROUPPER

mulAvxGFNI_9x4Xor_end:
	RET

// func mulAvxTwo_9x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x4Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 81 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x4Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), R10
	MOVQ  168(AX), R11
	MOVQ  192(AX), AX
	MOVQ  out_base+48(FP), R12
	MOVQ  (R12), R13
	MOVQ  24(R12), R14
	MOVQ  48(R12), R15
	MOVQ  72(R12), R12
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R13
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R12

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4
	MOVQ         n+80(FP), BP
	SHRQ         $0x05, BP

mulAvxTwo_9x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (R13), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU (R14), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU (R15), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU (R12), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU (R11), Y7
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1792(CX), Y5
	VMOVDQU 1824(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1856(CX), Y5
	VMOVDQU 1888(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1920(CX), Y5
	VMOVDQU 1952(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1984(CX), Y5
	VMOVDQU 2016(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU (AX), Y7
	ADDQ    $0x20, AX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 2048(CX), Y5
	VMOVDQU 2080(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 2112(CX), Y5
	VMOVDQU 2144(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 2176(CX), Y5
	VMOVDQU 2208(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 2240(CX), Y5
	VMOVDQU 2272(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	VMOVDQU Y0, (R13)
	ADDQ    $0x20, R13
	VMOVDQU Y1, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y2, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y3, (R12)
	ADDQ    $0x20, R12

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_9x4Xor_loop
	VZEROUPPER

mulAvxTwo_9x4Xor_end:
	RET

// func mulAvxTwo_9x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x5(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 100 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_9x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (R11), Y8
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU (R12), Y8
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2240(CX), Y6
	VMOVDQU 2272(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2304(CX), Y6
	VMOVDQU 2336(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2368(CX), Y6
	VMOVDQU 2400(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2432(CX), Y6
	VMOVDQU 2464(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2496(CX), Y6
	VMOVDQU 2528(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2560(CX), Y6
	VMOVDQU 2592(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2624(CX), Y6
	VMOVDQU 2656(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2688(CX), Y6
	VMOVDQU 2720(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2752(CX), Y6
	VMOVDQU 2784(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2816(CX), Y6
	VMOVDQU 2848(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x5_loop
	VZEROUPPER

mulAvxTwo_9x5_end:
	RET

// func mulGFNI_9x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x5_64(SB), $0-88
	// Loading 25 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 5 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x5_64_loop
	VZEROUPPER

mulGFNI_9x5_64_end:
	RET

// func mulAvxGFNI_9x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x5(SB), $0-88
	// Loading 9 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	MOVQ    (R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x5_loop
	VZEROUPPER

mulAvxGFNI_9x5_end:
	RET

// func mulGFNI_9x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x5_64Xor(SB), $0-88
	// Loading 25 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x5_64Xor_loop:
	// Load 5 outputs
	MOVQ      (R13), R15
	VMOVDQU64 (R15)(R14*1), Z25
	MOVQ      24(R13), R15
	VMOVDQU64 (R15)(R14*1), Z26
	MOVQ      48(R13), R15
	VMOVDQU64 (R15)(R14*1), Z27
	MOVQ      72(R13), R15
	VMOVDQU64 (R15)(R14*1), Z28
	MOVQ      96(R13), R15
	VMOVDQU64 (R15)(R14*1), Z29

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 5 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x5_64Xor_loop
	VZEROUPPER

mulGFNI_9x5_64Xor_end:
	RET

// func mulAvxGFNI_9x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x5Xor(SB), $0-88
	// Loading 9 of 45 tables to registers
	// Destination kept on stack
	// Full registers estimated 52 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x5Xor_loop:
	// Load 5 outputs
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y9
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y10
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y11
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y12
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	MOVQ    (R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x5Xor_loop
	VZEROUPPER

mulAvxGFNI_9x5Xor_end:
	RET

// func mulAvxTwo_9x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x5Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 100 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_9x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (R11), Y8
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU (R12), Y8
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2240(CX), Y6
	VMOVDQU 2272(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2304(CX), Y6
	VMOVDQU 2336(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2368(CX), Y6
	VMOVDQU 2400(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2432(CX), Y6
	VMOVDQU 2464(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2496(CX), Y6
	VMOVDQU 2528(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2560(CX), Y6
	VMOVDQU 2592(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2624(CX), Y6
	VMOVDQU 2656(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2688(CX), Y6
	VMOVDQU 2720(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2752(CX), Y6
	VMOVDQU 2784(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2816(CX), Y6
	VMOVDQU 2848(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x5Xor_loop
	VZEROUPPER

mulAvxTwo_9x5Xor_end:
	RET

// func mulAvxTwo_9x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x6(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 119 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_9x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R10), Y9
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (R11), Y9
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU (R12), Y9
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2688(CX), Y7
	VMOVDQU 2720(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2752(CX), Y7
	VMOVDQU 2784(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2816(CX), Y7
	VMOVDQU 2848(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2880(CX), Y7
	VMOVDQU 2912(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2944(CX), Y7
	VMOVDQU 2976(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3008(CX), Y7
	VMOVDQU 3040(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 3072(CX), Y7
	VMOVDQU 3104(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 3136(CX), Y7
	VMOVDQU 3168(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 3200(CX), Y7
	VMOVDQU 3232(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 3264(CX), Y7
	VMOVDQU 3296(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 3328(CX), Y7
	VMOVDQU 3360(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3392(CX), Y7
	VMOVDQU 3424(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x6_loop
	VZEROUPPER

mulAvxTwo_9x6_end:
	RET

// func mulGFNI_9x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x6_64(SB), $0-88
	// Loading 24 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 6 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x6_64_loop
	VZEROUPPER

mulGFNI_9x6_64_end:
	RET

// func mulAvxGFNI_9x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x6(SB), $0-88
	// Loading 8 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	MOVQ    (R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x6_loop
	VZEROUPPER

mulAvxGFNI_9x6_end:
	RET

// func mulGFNI_9x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x6_64Xor(SB), $0-88
	// Loading 24 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x6_64Xor_loop:
	// Load 6 outputs
	MOVQ      (R13), R15
	VMOVDQU64 (R15)(R14*1), Z24
	MOVQ      24(R13), R15
	VMOVDQU64 (R15)(R14*1), Z25
	MOVQ      48(R13), R15
	VMOVDQU64 (R15)(R14*1), Z26
	MOVQ      72(R13), R15
	VMOVDQU64 (R15)(R14*1), Z27
	MOVQ      96(R13), R15
	VMOVDQU64 (R15)(R14*1), Z28
	MOVQ      120(R13), R15
	VMOVDQU64 (R15)(R14*1), Z29

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 6 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x6_64Xor_loop
	VZEROUPPER

mulGFNI_9x6_64Xor_end:
	RET

// func mulAvxGFNI_9x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x6Xor(SB), $0-88
	// Loading 8 of 54 tables to registers
	// Destination kept on stack
	// Full registers estimated 62 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x6Xor_loop:
	// Load 6 outputs
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y8
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y9
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y10
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y11
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y12
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	MOVQ    (R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x6Xor_loop
	VZEROUPPER

mulAvxGFNI_9x6Xor_end:
	RET

// func mulAvxTwo_9x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x6Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 119 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_9x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R10), Y9
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (R11), Y9
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU (R12), Y9
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2688(CX), Y7
	VMOVDQU 2720(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2752(CX), Y7
	VMOVDQU 2784(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2816(CX), Y7
	VMOVDQU 2848(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2880(CX), Y7
	VMOVDQU 2912(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2944(CX), Y7
	VMOVDQU 2976(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3008(CX), Y7
	VMOVDQU 3040(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 3072(CX), Y7
	VMOVDQU 3104(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 3136(CX), Y7
	VMOVDQU 3168(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 3200(CX), Y7
	VMOVDQU 3232(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 3264(CX), Y7
	VMOVDQU 3296(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 3328(CX), Y7
	VMOVDQU 3360(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3392(CX), Y7
	VMOVDQU 3424(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x6Xor_loop
	VZEROUPPER

mulAvxTwo_9x6Xor_end:
	RET

// func mulAvxTwo_9x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x7(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 138 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_9x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (R11), Y10
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU (R12), Y10
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3136(CX), Y8
	VMOVDQU 3168(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3200(CX), Y8
	VMOVDQU 3232(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3264(CX), Y8
	VMOVDQU 3296(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3328(CX), Y8
	VMOVDQU 3360(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3392(CX), Y8
	VMOVDQU 3424(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3456(CX), Y8
	VMOVDQU 3488(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3520(CX), Y8
	VMOVDQU 3552(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3584(CX), Y8
	VMOVDQU 3616(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3648(CX), Y8
	VMOVDQU 3680(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3712(CX), Y8
	VMOVDQU 3744(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3776(CX), Y8
	VMOVDQU 3808(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3840(CX), Y8
	VMOVDQU 3872(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3904(CX), Y8
	VMOVDQU 3936(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3968(CX), Y8
	VMOVDQU 4000(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x7_loop
	VZEROUPPER

mulAvxTwo_9x7_end:
	RET

// func mulGFNI_9x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x7_64(SB), $0-88
	// Loading 23 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 7 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x7_64_loop
	VZEROUPPER

mulGFNI_9x7_64_end:
	RET

// func mulAvxGFNI_9x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x7(SB), $0-88
	// Loading 7 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x7_loop
	VZEROUPPER

mulAvxGFNI_9x7_end:
	RET

// func mulGFNI_9x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x7_64Xor(SB), $0-88
	// Loading 23 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x7_64Xor_loop:
	// Load 7 outputs
	MOVQ      (R13), R15
	VMOVDQU64 (R15)(R14*1), Z23
	MOVQ      24(R13), R15
	VMOVDQU64 (R15)(R14*1), Z24
	MOVQ      48(R13), R15
	VMOVDQU64 (R15)(R14*1), Z25
	MOVQ      72(R13), R15
	VMOVDQU64 (R15)(R14*1), Z26
	MOVQ      96(R13), R15
	VMOVDQU64 (R15)(R14*1), Z27
	MOVQ      120(R13), R15
	VMOVDQU64 (R15)(R14*1), Z28
	MOVQ      144(R13), R15
	VMOVDQU64 (R15)(R14*1), Z29

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 7 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x7_64Xor_loop
	VZEROUPPER

mulGFNI_9x7_64Xor_end:
	RET

// func mulAvxGFNI_9x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x7Xor(SB), $0-88
	// Loading 7 of 63 tables to registers
	// Destination kept on stack
	// Full registers estimated 72 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x7Xor_loop:
	// Load 7 outputs
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y7
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y8
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y9
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y10
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y11
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y12
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x7Xor_loop
	VZEROUPPER

mulAvxGFNI_9x7Xor_end:
	RET

// func mulAvxTwo_9x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x7Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 138 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_9x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (R11), Y10
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU (R12), Y10
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3136(CX), Y8
	VMOVDQU 3168(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3200(CX), Y8
	VMOVDQU 3232(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3264(CX), Y8
	VMOVDQU 3296(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3328(CX), Y8
	VMOVDQU 3360(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3392(CX), Y8
	VMOVDQU 3424(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3456(CX), Y8
	VMOVDQU 3488(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3520(CX), Y8
	VMOVDQU 3552(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3584(CX), Y8
	VMOVDQU 3616(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3648(CX), Y8
	VMOVDQU 3680(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3712(CX), Y8
	VMOVDQU 3744(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3776(CX), Y8
	VMOVDQU 3808(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3840(CX), Y8
	VMOVDQU 3872(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3904(CX), Y8
	VMOVDQU 3936(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3968(CX), Y8
	VMOVDQU 4000(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x7Xor_loop
	VZEROUPPER

mulAvxTwo_9x7Xor_end:
	RET

// func mulAvxTwo_9x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x8(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 157 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_9x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (R11), Y11
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU (R12), Y11
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3584(CX), Y9
	VMOVDQU 3616(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3648(CX), Y9
	VMOVDQU 3680(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3712(CX), Y9
	VMOVDQU 3744(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3776(CX), Y9
	VMOVDQU 3808(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3840(CX), Y9
	VMOVDQU 3872(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3904(CX), Y9
	VMOVDQU 3936(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3968(CX), Y9
	VMOVDQU 4000(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4032(CX), Y9
	VMOVDQU 4064(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 4096(CX), Y9
	VMOVDQU 4128(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 4160(CX), Y9
	VMOVDQU 4192(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 4224(CX), Y9
	VMOVDQU 4256(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 4288(CX), Y9
	VMOVDQU 4320(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 4352(CX), Y9
	VMOVDQU 4384(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 4416(CX), Y9
	VMOVDQU 4448(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 4480(CX), Y9
	VMOVDQU 4512(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4544(CX), Y9
	VMOVDQU 4576(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y7, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x8_loop
	VZEROUPPER

mulAvxTwo_9x8_end:
	RET

// func mulGFNI_9x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x8_64(SB), $0-88
	// Loading 22 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 8 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z22, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      168(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x8_64_loop
	VZEROUPPER

mulGFNI_9x8_64_end:
	RET

// func mulAvxGFNI_9x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x8(SB), $0-88
	// Loading 6 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x8_loop
	VZEROUPPER

mulAvxGFNI_9x8_end:
	RET

// func mulGFNI_9x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x8_64Xor(SB), $0-88
	// Loading 22 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x8_64Xor_loop:
	// Load 8 outputs
	MOVQ      (R13), R15
	VMOVDQU64 (R15)(R14*1), Z22
	MOVQ      24(R13), R15
	VMOVDQU64 (R15)(R14*1), Z23
	MOVQ      48(R13), R15
	VMOVDQU64 (R15)(R14*1), Z24
	MOVQ      72(R13), R15
	VMOVDQU64 (R15)(R14*1), Z25
	MOVQ      96(R13), R15
	VMOVDQU64 (R15)(R14*1), Z26
	MOVQ      120(R13), R15
	VMOVDQU64 (R15)(R14*1), Z27
	MOVQ      144(R13), R15
	VMOVDQU64 (R15)(R14*1), Z28
	MOVQ      168(R13), R15
	VMOVDQU64 (R15)(R14*1), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 8 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z22, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      168(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x8_64Xor_loop
	VZEROUPPER

mulGFNI_9x8_64Xor_end:
	RET

// func mulAvxGFNI_9x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x8Xor(SB), $0-88
	// Loading 6 of 72 tables to registers
	// Destination kept on stack
	// Full registers estimated 82 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x8Xor_loop:
	// Load 8 outputs
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y6
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y7
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y8
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y9
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y10
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y11
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y12
	MOVQ    168(R13), R15
	VMOVDQU (R15)(R14*1), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x8Xor_loop
	VZEROUPPER

mulAvxGFNI_9x8Xor_end:
	RET

// func mulAvxTwo_9x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x8Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 157 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_9x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	MOVQ    168(R13), R15
	VMOVDQU (R15)(R14*1), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (R11), Y11
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU (R12), Y11
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3584(CX), Y9
	VMOVDQU 3616(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3648(CX), Y9
	VMOVDQU 3680(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3712(CX), Y9
	VMOVDQU 3744(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3776(CX), Y9
	VMOVDQU 3808(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3840(CX), Y9
	VMOVDQU 3872(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3904(CX), Y9
	VMOVDQU 3936(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3968(CX), Y9
	VMOVDQU 4000(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4032(CX), Y9
	VMOVDQU 4064(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 4096(CX), Y9
	VMOVDQU 4128(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 4160(CX), Y9
	VMOVDQU 4192(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 4224(CX), Y9
	VMOVDQU 4256(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 4288(CX), Y9
	VMOVDQU 4320(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 4352(CX), Y9
	VMOVDQU 4384(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 4416(CX), Y9
	VMOVDQU 4448(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 4480(CX), Y9
	VMOVDQU 4512(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4544(CX), Y9
	VMOVDQU 4576(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y7, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x8Xor_loop
	VZEROUPPER

mulAvxTwo_9x8Xor_end:
	RET

// func mulAvxTwo_9x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x9(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 176 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_9x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (R11), Y12
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU (R12), Y12
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4032(CX), Y10
	VMOVDQU 4064(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4096(CX), Y10
	VMOVDQU 4128(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4160(CX), Y10
	VMOVDQU 4192(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4224(CX), Y10
	VMOVDQU 4256(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4288(CX), Y10
	VMOVDQU 4320(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4352(CX), Y10
	VMOVDQU 4384(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4416(CX), Y10
	VMOVDQU 4448(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 4480(CX), Y10
	VMOVDQU 4512(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 4544(CX), Y10
	VMOVDQU 4576(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4608(CX), Y10
	VMOVDQU 4640(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4672(CX), Y10
	VMOVDQU 4704(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4736(CX), Y10
	VMOVDQU 4768(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4800(CX), Y10
	VMOVDQU 4832(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4864(CX), Y10
	VMOVDQU 4896(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4928(CX), Y10
	VMOVDQU 4960(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4992(CX), Y10
	VMOVDQU 5024(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 5056(CX), Y10
	VMOVDQU 5088(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 5120(CX), Y10
	VMOVDQU 5152(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y8, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x9_loop
	VZEROUPPER

mulAvxTwo_9x9_end:
	RET

// func mulGFNI_9x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x9_64(SB), $0-88
	// Loading 21 of 81 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 9 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z21, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z22, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      168(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      192(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x9_64_loop
	VZEROUPPER

mulGFNI_9x9_64_end:
	RET

// func mulAvxGFNI_9x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x9(SB), $0-88
	// Loading 5 of 81 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x9_loop
	VZEROUPPER

mulAvxGFNI_9x9_end:
	RET

// func mulGFNI_9x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x9_64Xor(SB), $0-88
	// Loading 21 of 81 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x9_64Xor_loop:
	// Load 9 outputs
	MOVQ      (R13), R15
	VMOVDQU64 (R15)(R14*1), Z21
	MOVQ      24(R13), R15
	VMOVDQU64 (R15)(R14*1), Z22
	MOVQ      48(R13), R15
	VMOVDQU64 (R15)(R14*1), Z23
	MOVQ      72(R13), R15
	VMOVDQU64 (R15)(R14*1), Z24
	MOVQ      96(R13), R15
	VMOVDQU64 (R15)(R14*1), Z25
	MOVQ      120(R13), R15
	VMOVDQU64 (R15)(R14*1), Z26
	MOVQ      144(R13), R15
	VMOVDQU64 (R15)(R14*1), Z27
	MOVQ      168(R13), R15
	VMOVDQU64 (R15)(R14*1), Z28
	MOVQ      192(R13), R15
	VMOVDQU64 (R15)(R14*1), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 9 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z21, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z22, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      168(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      192(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x9_64Xor_loop
	VZEROUPPER

mulGFNI_9x9_64Xor_end:
	RET

// func mulAvxGFNI_9x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x9Xor(SB), $0-88
	// Loading 5 of 81 tables to registers
	// Destination kept on stack
	// Full registers estimated 92 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x9Xor_loop:
	// Load 9 outputs
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y5
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y6
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y7
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y8
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y9
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y10
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y11
	MOVQ    168(R13), R15
	VMOVDQU (R15)(R14*1), Y12
	MOVQ    192(R13), R15
	VMOVDQU (R15)(R14*1), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x9Xor_loop
	VZEROUPPER

mulAvxGFNI_9x9Xor_end:
	RET

// func mulAvxTwo_9x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x9Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 176 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_9x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	MOVQ    168(R13), R15
	VMOVDQU (R15)(R14*1), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	MOVQ    192(R13), R15
	VMOVDQU (R15)(R14*1), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (R11), Y12
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU (R12), Y12
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4032(CX), Y10
	VMOVDQU 4064(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4096(CX), Y10
	VMOVDQU 4128(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4160(CX), Y10
	VMOVDQU 4192(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4224(CX), Y10
	VMOVDQU 4256(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4288(CX), Y10
	VMOVDQU 4320(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4352(CX), Y10
	VMOVDQU 4384(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4416(CX), Y10
	VMOVDQU 4448(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 4480(CX), Y10
	VMOVDQU 4512(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 4544(CX), Y10
	VMOVDQU 4576(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4608(CX), Y10
	VMOVDQU 4640(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4672(CX), Y10
	VMOVDQU 4704(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4736(CX), Y10
	VMOVDQU 4768(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4800(CX), Y10
	VMOVDQU 4832(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4864(CX), Y10
	VMOVDQU 4896(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4928(CX), Y10
	VMOVDQU 4960(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4992(CX), Y10
	VMOVDQU 5024(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 5056(CX), Y10
	VMOVDQU 5088(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 5120(CX), Y10
	VMOVDQU 5152(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y8, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x9Xor_loop
	VZEROUPPER

mulAvxTwo_9x9Xor_end:
	RET

// func mulAvxTwo_9x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x10(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 195 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_9x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (R11), Y13
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU (R12), Y13
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 4480(CX), Y11
	VMOVDQU 4512(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 4544(CX), Y11
	VMOVDQU 4576(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 4608(CX), Y11
	VMOVDQU 4640(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4672(CX), Y11
	VMOVDQU 4704(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4736(CX), Y11
	VMOVDQU 4768(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4800(CX), Y11
	VMOVDQU 4832(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4864(CX), Y11
	VMOVDQU 4896(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4928(CX), Y11
	VMOVDQU 4960(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4992(CX), Y11
	VMOVDQU 5024(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5056(CX), Y11
	VMOVDQU 5088(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 5120(CX), Y11
	VMOVDQU 5152(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 5184(CX), Y11
	VMOVDQU 5216(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 5248(CX), Y11
	VMOVDQU 5280(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 5312(CX), Y11
	VMOVDQU 5344(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 5376(CX), Y11
	VMOVDQU 5408(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 5440(CX), Y11
	VMOVDQU 5472(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 5504(CX), Y11
	VMOVDQU 5536(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 5568(CX), Y11
	VMOVDQU 5600(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 5632(CX), Y11
	VMOVDQU 5664(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5696(CX), Y11
	VMOVDQU 5728(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    216(R13), R15
	VMOVDQU Y9, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x10_loop
	VZEROUPPER

mulAvxTwo_9x10_end:
	RET

// func mulGFNI_9x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x10_64(SB), $0-88
	// Loading 20 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 102 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 10 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z20, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z21, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z22, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      168(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      192(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      216(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x10_64_loop
	VZEROUPPER

mulGFNI_9x10_64_end:
	RET

// func mulAvxGFNI_9x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x10(SB), $0-88
	// Loading 4 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 102 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   648(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   656(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   664(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   672(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   680(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   688(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   696(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   704(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   712(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    216(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x10_loop
	VZEROUPPER

mulAvxGFNI_9x10_end:
	RET

// func mulGFNI_9x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_9x10_64Xor(SB), $0-88
	// Loading 20 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 102 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_9x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), DX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulGFNI_9x10_64Xor_loop:
	// Load 10 outputs
	MOVQ      (R13), R15
	VMOVDQU64 (R15)(R14*1), Z20
	MOVQ      24(R13), R15
	VMOVDQU64 (R15)(R14*1), Z21
	MOVQ      48(R13), R15
	VMOVDQU64 (R15)(R14*1), Z22
	MOVQ      72(R13), R15
	VMOVDQU64 (R15)(R14*1), Z23
	MOVQ      96(R13), R15
	VMOVDQU64 (R15)(R14*1), Z24
	MOVQ      120(R13), R15
	VMOVDQU64 (R15)(R14*1), Z25
	MOVQ      144(R13), R15
	VMOVDQU64 (R15)(R14*1), Z26
	MOVQ      168(R13), R15
	VMOVDQU64 (R15)(R14*1), Z27
	MOVQ      192(R13), R15
	VMOVDQU64 (R15)(R14*1), Z28
	MOVQ      216(R13), R15
	VMOVDQU64 (R15)(R14*1), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 10 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R13), R15
	VMOVDQU64 Z20, (R15)(R14*1)
	MOVQ      24(R13), R15
	VMOVDQU64 Z21, (R15)(R14*1)
	MOVQ      48(R13), R15
	VMOVDQU64 Z22, (R15)(R14*1)
	MOVQ      72(R13), R15
	VMOVDQU64 Z23, (R15)(R14*1)
	MOVQ      96(R13), R15
	VMOVDQU64 Z24, (R15)(R14*1)
	MOVQ      120(R13), R15
	VMOVDQU64 Z25, (R15)(R14*1)
	MOVQ      144(R13), R15
	VMOVDQU64 Z26, (R15)(R14*1)
	MOVQ      168(R13), R15
	VMOVDQU64 Z27, (R15)(R14*1)
	MOVQ      192(R13), R15
	VMOVDQU64 Z28, (R15)(R14*1)
	MOVQ      216(R13), R15
	VMOVDQU64 Z29, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x40, R14
	DECQ AX
	JNZ  mulGFNI_9x10_64Xor_loop
	VZEROUPPER

mulGFNI_9x10_64Xor_end:
	RET

// func mulAvxGFNI_9x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_9x10Xor(SB), $0-88
	// Loading 4 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 102 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_9x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), DX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         start+72(FP), R14

	// Add start offset to input
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, DX

mulAvxGFNI_9x10Xor_loop:
	// Load 10 outputs
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y4
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y5
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y6
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y7
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y8
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y9
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y10
	MOVQ    168(R13), R15
	VMOVDQU (R15)(R14*1), Y11
	MOVQ    192(R13), R15
	VMOVDQU (R15)(R14*1), Y12
	MOVQ    216(R13), R15
	VMOVDQU (R15)(R14*1), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   648(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   656(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   664(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   672(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   680(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   688(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   696(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   704(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   712(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y9, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y10, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y11, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y12, (R15)(R14*1)
	MOVQ    216(R13), R15
	VMOVDQU Y13, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxGFNI_9x10Xor_loop
	VZEROUPPER

mulAvxGFNI_9x10Xor_end:
	RET

// func mulAvxTwo_9x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_9x10Xor(SB), NOSPLIT, $0-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 195 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_9x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), DX
	MOVQ  out_base+48(FP), R13
	MOVQ  start+72(FP), R14

	// Add start offset to input
	ADDQ         R14, BX
	ADDQ         R14, SI
	ADDQ         R14, DI
	ADDQ         R14, R8
	ADDQ         R14, R9
	ADDQ         R14, R10
	ADDQ         R14, R11
	ADDQ         R14, R12
	ADDQ         R14, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_9x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	MOVQ    (R13), R15
	VMOVDQU (R15)(R14*1), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	MOVQ    24(R13), R15
	VMOVDQU (R15)(R14*1), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	MOVQ    48(R13), R15
	VMOVDQU (R15)(R14*1), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	MOVQ    72(R13), R15
	VMOVDQU (R15)(R14*1), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	MOVQ    96(R13), R15
	VMOVDQU (R15)(R14*1), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	MOVQ    120(R13), R15
	VMOVDQU (R15)(R14*1), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	MOVQ    144(R13), R15
	VMOVDQU (R15)(R14*1), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	MOVQ    168(R13), R15
	VMOVDQU (R15)(R14*1), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	MOVQ    192(R13), R15
	VMOVDQU (R15)(R14*1), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	MOVQ    216(R13), R15
	VMOVDQU (R15)(R14*1), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (R11), Y13
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU (R12), Y13
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 4480(CX), Y11
	VMOVDQU 4512(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 4544(CX), Y11
	VMOVDQU 4576(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 4608(CX), Y11
	VMOVDQU 4640(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4672(CX), Y11
	VMOVDQU 4704(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4736(CX), Y11
	VMOVDQU 4768(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4800(CX), Y11
	VMOVDQU 4832(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4864(CX), Y11
	VMOVDQU 4896(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4928(CX), Y11
	VMOVDQU 4960(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4992(CX), Y11
	VMOVDQU 5024(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5056(CX), Y11
	VMOVDQU 5088(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 5120(CX), Y11
	VMOVDQU 5152(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 5184(CX), Y11
	VMOVDQU 5216(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 5248(CX), Y11
	VMOVDQU 5280(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 5312(CX), Y11
	VMOVDQU 5344(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 5376(CX), Y11
	VMOVDQU 5408(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 5440(CX), Y11
	VMOVDQU 5472(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 5504(CX), Y11
	VMOVDQU 5536(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 5568(CX), Y11
	VMOVDQU 5600(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 5632(CX), Y11
	VMOVDQU 5664(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5696(CX), Y11
	VMOVDQU 5728(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R13), R15
	VMOVDQU Y0, (R15)(R14*1)
	MOVQ    24(R13), R15
	VMOVDQU Y1, (R15)(R14*1)
	MOVQ    48(R13), R15
	VMOVDQU Y2, (R15)(R14*1)
	MOVQ    72(R13), R15
	VMOVDQU Y3, (R15)(R14*1)
	MOVQ    96(R13), R15
	VMOVDQU Y4, (R15)(R14*1)
	MOVQ    120(R13), R15
	VMOVDQU Y5, (R15)(R14*1)
	MOVQ    144(R13), R15
	VMOVDQU Y6, (R15)(R14*1)
	MOVQ    168(R13), R15
	VMOVDQU Y7, (R15)(R14*1)
	MOVQ    192(R13), R15
	VMOVDQU Y8, (R15)(R14*1)
	MOVQ    216(R13), R15
	VMOVDQU Y9, (R15)(R14*1)

	// Prepare for next loop
	ADDQ $0x20, R14
	DECQ AX
	JNZ  mulAvxTwo_9x10Xor_loop
	VZEROUPPER

mulAvxTwo_9x10Xor_end:
	RET

// func mulAvxTwo_10x1_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x1_64(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 46 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x1_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  (R14), R14
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_10x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	VPXOR   Y3, Y4, Y0
	VPXOR   Y5, Y6, Y1

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (R11), Y6
	VMOVDQU 32(R11), Y5
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU (R12), Y6
	VMOVDQU 32(R12), Y5
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 448(CX), Y3
	VMOVDQU 480(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU (R13), Y6
	VMOVDQU 32(R13), Y5
	ADDQ    $0x40, R13
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 512(CX), Y3
	VMOVDQU 544(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 9 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 576(CX), Y3
	VMOVDQU 608(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_10x1_64_loop
	VZEROUPPER

mulAvxTwo_10x1_64_end:
	RET

// func mulGFNI_10x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x1_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 13 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x1_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), R12
	MOVQ            216(CX), CX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            (R13), R13
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, CX

mulGFNI_10x1_64_loop:
	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z11
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z11, Z10

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z11
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z11
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z11
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z11
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z11
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (R10), Z11
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z6, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU64      (R11), Z11
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z7, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU64      (R12), Z11
	ADDQ           $0x40, R12
	VGF2P8AFFINEQB $0x00, Z8, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 9 to 1 outputs
	VMOVDQU64      (CX), Z11
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z9, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Store 1 outputs
	VMOVDQU64 Z10, (R13)
	ADDQ      $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_10x1_64_loop
	VZEROUPPER

mulGFNI_10x1_64_end:
	RET

// func mulAvxGFNI_10x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x1(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 13 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x1_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), R10
	MOVQ         168(CX), R11
	MOVQ         192(CX), R12
	MOVQ         216(CX), CX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R13
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, CX

mulAvxGFNI_10x1_loop:
	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y11
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y11, Y10

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y11
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y11
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y11
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y11
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y11
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (R10), Y11
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y6, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 7 to 1 outputs
	VMOVDQU        (R11), Y11
	ADDQ           $0x20, R11
	VGF2P8AFFINEQB $0x00, Y7, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 8 to 1 outputs
	VMOVDQU        (R12), Y11
	ADDQ           $0x20, R12
	VGF2P8AFFINEQB $0x00, Y8, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 9 to 1 outputs
	VMOVDQU        (CX), Y11
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y9, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Store 1 outputs
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_10x1_loop
	VZEROUPPER

mulAvxGFNI_10x1_end:
	RET

// func mulGFNI_10x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x1_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 13 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x1_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), R12
	MOVQ            216(CX), CX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            (R13), R13
	MOVQ            start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, CX

mulGFNI_10x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU64 (R13), Z10

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU64      (DX), Z11
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU64      (BX), Z11
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z1, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU64      (SI), Z11
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z2, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU64      (DI), Z11
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z3, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU64      (R8), Z11
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z4, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU64      (R9), Z11
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z5, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU64      (R10), Z11
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z6, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU64      (R11), Z11
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z7, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU64      (R12), Z11
	ADDQ           $0x40, R12
	VGF2P8AFFINEQB $0x00, Z8, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Load and process 64 bytes from input 9 to 1 outputs
	VMOVDQU64      (CX), Z11
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z9, Z11, Z11
	VXORPD         Z10, Z11, Z10

	// Store 1 outputs
	VMOVDQU64 Z10, (R13)
	ADDQ      $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_10x1_64Xor_loop
	VZEROUPPER

mulGFNI_10x1_64Xor_end:
	RET

// func mulAvxGFNI_10x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x1Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 13 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x1Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), CX
	MOVQ         (CX), DX
	MOVQ         24(CX), BX
	MOVQ         48(CX), SI
	MOVQ         72(CX), DI
	MOVQ         96(CX), R8
	MOVQ         120(CX), R9
	MOVQ         144(CX), R10
	MOVQ         168(CX), R11
	MOVQ         192(CX), R12
	MOVQ         216(CX), CX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R13
	MOVQ         start+72(FP), R14

	// Add start offset to output
	ADDQ R14, R13

	// Add start offset to input
	ADDQ R14, DX
	ADDQ R14, BX
	ADDQ R14, SI
	ADDQ R14, DI
	ADDQ R14, R8
	ADDQ R14, R9
	ADDQ R14, R10
	ADDQ R14, R11
	ADDQ R14, R12
	ADDQ R14, CX

mulAvxGFNI_10x1Xor_loop:
	// Load 1 outputs
	VMOVDQU (R13), Y10

	// Load and process 32 bytes from input 0 to 1 outputs
	VMOVDQU        (DX), Y11
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 1 to 1 outputs
	VMOVDQU        (BX), Y11
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y1, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 2 to 1 outputs
	VMOVDQU        (SI), Y11
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 3 to 1 outputs
	VMOVDQU        (DI), Y11
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y3, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 4 to 1 outputs
	VMOVDQU        (R8), Y11
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y4, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 5 to 1 outputs
	VMOVDQU        (R9), Y11
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y5, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 6 to 1 outputs
	VMOVDQU        (R10), Y11
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y6, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 7 to 1 outputs
	VMOVDQU        (R11), Y11
	ADDQ           $0x20, R11
	VGF2P8AFFINEQB $0x00, Y7, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 8 to 1 outputs
	VMOVDQU        (R12), Y11
	ADDQ           $0x20, R12
	VGF2P8AFFINEQB $0x00, Y8, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Load and process 32 bytes from input 9 to 1 outputs
	VMOVDQU        (CX), Y11
	ADDQ           $0x20, CX
	VGF2P8AFFINEQB $0x00, Y9, Y11, Y11
	VXORPD         Y10, Y11, Y10

	// Store 1 outputs
	VMOVDQU Y10, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_10x1Xor_loop
	VZEROUPPER

mulAvxGFNI_10x1Xor_end:
	RET

// func mulAvxTwo_10x1_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x1_64Xor(SB), $0-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 46 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x1_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  (R14), R14
	MOVQ  start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, R15
	MOVQ         R15, X2
	VPBROADCASTB X2, Y2

mulAvxTwo_10x1_64Xor_loop:
	// Load 1 outputs
	VMOVDQU (R14), Y0
	VMOVDQU 32(R14), Y1

	// Load and process 64 bytes from input 0 to 1 outputs
	VMOVDQU (BX), Y6
	VMOVDQU 32(BX), Y5
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 1 to 1 outputs
	VMOVDQU (SI), Y6
	VMOVDQU 32(SI), Y5
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 2 to 1 outputs
	VMOVDQU (DI), Y6
	VMOVDQU 32(DI), Y5
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 128(CX), Y3
	VMOVDQU 160(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 3 to 1 outputs
	VMOVDQU (R8), Y6
	VMOVDQU 32(R8), Y5
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 192(CX), Y3
	VMOVDQU 224(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 4 to 1 outputs
	VMOVDQU (R9), Y6
	VMOVDQU 32(R9), Y5
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 256(CX), Y3
	VMOVDQU 288(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 5 to 1 outputs
	VMOVDQU (R10), Y6
	VMOVDQU 32(R10), Y5
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 320(CX), Y3
	VMOVDQU 352(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 6 to 1 outputs
	VMOVDQU (R11), Y6
	VMOVDQU 32(R11), Y5
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 384(CX), Y3
	VMOVDQU 416(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 7 to 1 outputs
	VMOVDQU (R12), Y6
	VMOVDQU 32(R12), Y5
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 448(CX), Y3
	VMOVDQU 480(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 8 to 1 outputs
	VMOVDQU (R13), Y6
	VMOVDQU 32(R13), Y5
	ADDQ    $0x40, R13
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 512(CX), Y3
	VMOVDQU 544(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Load and process 64 bytes from input 9 to 1 outputs
	VMOVDQU (DX), Y6
	VMOVDQU 32(DX), Y5
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y6, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPAND   Y2, Y6, Y6
	VPAND   Y2, Y5, Y5
	VPAND   Y2, Y7, Y7
	VPAND   Y2, Y8, Y8
	VMOVDQU 576(CX), Y3
	VMOVDQU 608(CX), Y4
	VPSHUFB Y5, Y3, Y5
	VPSHUFB Y6, Y3, Y3
	VPSHUFB Y8, Y4, Y6
	VPSHUFB Y7, Y4, Y4
	XOR3WAY( $0x00, Y3, Y4, Y0)
	XOR3WAY( $0x00, Y5, Y6, Y1)

	// Store 1 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_10x1_64Xor_loop
	VZEROUPPER

mulAvxTwo_10x1_64Xor_end:
	RET

// func mulAvxTwo_10x2_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x2_64(SB), $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 89 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x2_64_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  (R14), R15
	MOVQ  24(R14), R14
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R15
	ADDQ BP, R14

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, R12
	ADDQ         BP, R13
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_10x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VPXOR   Y7, Y8, Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VPXOR   Y7, Y8, Y3

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (R11), Y9
	VMOVDQU 32(R11), Y11
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU (R12), Y9
	VMOVDQU 32(R12), Y11
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU (R13), Y9
	VMOVDQU 32(R13), Y11
	ADDQ    $0x40, R13
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 9 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R15)
	VMOVDQU Y1, 32(R15)
	ADDQ    $0x40, R15
	VMOVDQU Y2, (R14)
	VMOVDQU Y3, 32(R14)
	ADDQ    $0x40, R14

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_10x2_64_loop
	VZEROUPPER

mulAvxTwo_10x2_64_end:
	RET

// func mulGFNI_10x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x2_64(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 24 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x2_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), R12
	MOVQ            216(CX), CX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            (R13), R14
	MOVQ            24(R13), R13
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14
	ADDQ R15, R13

	// Add start offset to input
	ADDQ R15, DX
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, CX

mulGFNI_10x2_64_loop:
	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z22
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z22, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z22, Z21

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z22
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z3, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z22
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z5, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z22
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z7, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z22
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z9, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z22
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (R10), Z22
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z12, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z13, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU64      (R11), Z22
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z14, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z15, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU64      (R12), Z22
	ADDQ           $0x40, R12
	VGF2P8AFFINEQB $0x00, Z16, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z17, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 9 to 2 outputs
	VMOVDQU64      (CX), Z22
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z18, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z19, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Store 2 outputs
	VMOVDQU64 Z20, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z21, (R13)
	ADDQ      $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_10x2_64_loop
	VZEROUPPER

mulGFNI_10x2_64_end:
	RET

// func mulAvxGFNI_10x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x2(SB), $8-88
	// Loading 12 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 24 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x2_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         (R14), R15
	MOVQ         24(R14), R14
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R15
	ADDQ BP, R14

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, DX

mulAvxGFNI_10x2_loop:
	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 2 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 2 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R14)
	ADDQ    $0x20, R14

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_10x2_loop
	VZEROUPPER

mulAvxGFNI_10x2_end:
	RET

// func mulGFNI_10x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x2_64Xor(SB), $0-88
	// Loading all tables to registers
	// Destination kept in GP registers
	// Full registers estimated 24 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x2_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), CX
	MOVQ            (CX), DX
	MOVQ            24(CX), BX
	MOVQ            48(CX), SI
	MOVQ            72(CX), DI
	MOVQ            96(CX), R8
	MOVQ            120(CX), R9
	MOVQ            144(CX), R10
	MOVQ            168(CX), R11
	MOVQ            192(CX), R12
	MOVQ            216(CX), CX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            (R13), R14
	MOVQ            24(R13), R13
	MOVQ            start+72(FP), R15

	// Add start offset to output
	ADDQ R15, R14
	ADDQ R15, R13

	// Add start offset to input
	ADDQ R15, DX
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, CX

mulGFNI_10x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU64 (R14), Z20
	VMOVDQU64 (R13), Z21

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU64      (DX), Z22
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU64      (BX), Z22
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z2, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z3, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU64      (SI), Z22
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z5, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU64      (DI), Z22
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z6, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z7, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU64      (R8), Z22
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z8, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z9, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU64      (R9), Z22
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z10, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU64      (R10), Z22
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z12, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z13, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU64      (R11), Z22
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z14, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z15, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU64      (R12), Z22
	ADDQ           $0x40, R12
	VGF2P8AFFINEQB $0x00, Z16, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z17, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Load and process 64 bytes from input 9 to 2 outputs
	VMOVDQU64      (CX), Z22
	ADDQ           $0x40, CX
	VGF2P8AFFINEQB $0x00, Z18, Z22, Z23
	VXORPD         Z20, Z23, Z20
	VGF2P8AFFINEQB $0x00, Z19, Z22, Z23
	VXORPD         Z21, Z23, Z21

	// Store 2 outputs
	VMOVDQU64 Z20, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z21, (R13)
	ADDQ      $0x40, R13

	// Prepare for next loop
	DECQ AX
	JNZ  mulGFNI_10x2_64Xor_loop
	VZEROUPPER

mulGFNI_10x2_64Xor_end:
	RET

// func mulAvxGFNI_10x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x2Xor(SB), $8-88
	// Loading 12 of 20 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 24 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x2Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	VBROADCASTSD 88(CX), Y11
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         (R14), R15
	MOVQ         24(R14), R14
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R15
	ADDQ BP, R14

	// Add start offset to input
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, R13
	ADDQ BP, DX

mulAvxGFNI_10x2Xor_loop:
	// Load 2 outputs
	VMOVDQU (R15), Y12
	VMOVDQU (R14), Y13

	// Load and process 32 bytes from input 0 to 2 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 2 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 2 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 2 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 2 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 2 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 2 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 2 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 2 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 2 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 2 outputs
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R14)
	ADDQ    $0x20, R14

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxGFNI_10x2Xor_loop
	VZEROUPPER

mulAvxGFNI_10x2Xor_end:
	RET

// func mulAvxTwo_10x2_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x2_64Xor(SB), $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 89 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x2_64Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  (R14), R15
	MOVQ  24(R14), R14
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R15
	ADDQ BP, R14

	// Add start offset to input
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, R12
	ADDQ         BP, R13
	ADDQ         BP, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_10x2_64Xor_loop:
	// Load 2 outputs
	VMOVDQU (R15), Y0
	VMOVDQU 32(R15), Y1
	VMOVDQU (R14), Y2
	VMOVDQU 32(R14), Y3

	// Load and process 64 bytes from input 0 to 2 outputs
	VMOVDQU (BX), Y9
	VMOVDQU 32(BX), Y11
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 1 to 2 outputs
	VMOVDQU (SI), Y9
	VMOVDQU 32(SI), Y11
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 2 to 2 outputs
	VMOVDQU (DI), Y9
	VMOVDQU 32(DI), Y11
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 3 to 2 outputs
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y11
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 4 to 2 outputs
	VMOVDQU (R9), Y9
	VMOVDQU 32(R9), Y11
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 5 to 2 outputs
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y11
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 6 to 2 outputs
	VMOVDQU (R11), Y9
	VMOVDQU 32(R11), Y11
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 7 to 2 outputs
	VMOVDQU (R12), Y9
	VMOVDQU 32(R12), Y11
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 8 to 2 outputs
	VMOVDQU (R13), Y9
	VMOVDQU 32(R13), Y11
	ADDQ    $0x40, R13
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Load and process 64 bytes from input 9 to 2 outputs
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y11
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y9, Y10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y4, Y9, Y9
	VPAND   Y4, Y11, Y11
	VPAND   Y4, Y10, Y10
	VPAND   Y4, Y12, Y12
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y11, Y5, Y7
	VPSHUFB Y9, Y5, Y5
	VPSHUFB Y12, Y6, Y8
	VPSHUFB Y10, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// Store 2 outputs
	VMOVDQU Y0, (R15)
	VMOVDQU Y1, 32(R15)
	ADDQ    $0x40, R15
	VMOVDQU Y2, (R14)
	VMOVDQU Y3, 32(R14)
	ADDQ    $0x40, R14

	// Prepare for next loop
	DECQ AX
	JNZ  mulAvxTwo_10x2_64Xor_loop
	VZEROUPPER

mulAvxTwo_10x2_64Xor_end:
	RET

// func mulAvxTwo_10x3_64(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x3_64(SB), $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 130 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x3_64_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), R10
	MOVQ  168(AX), R11
	MOVQ  192(AX), R12
	MOVQ  216(AX), AX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R14
	MOVQ  24(R13), R15
	MOVQ  48(R13), R13
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, R12
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulAvxTwo_10x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VPXOR   Y9, Y10, Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VPXOR   Y9, Y10, Y3
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VPXOR   Y9, Y10, Y5

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU (R11), Y11
	VMOVDQU 32(R11), Y13
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU (R12), Y11
	VMOVDQU 32(R12), Y13
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 9 to 3 outputs
	VMOVDQU (AX), Y11
	VMOVDQU 32(AX), Y13
	ADDQ    $0x40, AX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y2, (R15)
	VMOVDQU Y3, 32(R15)
	ADDQ    $0x40, R15
	VMOVDQU Y4, (R13)
	VMOVDQU Y5, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_10x3_64_loop
	VZEROUPPER

mulAvxTwo_10x3_64_end:
	RET

// func mulGFNI_10x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x3_64(SB), $8-88
	// Loading 27 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x3_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	VBROADCASTF32X2 208(CX), Z26
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), R10
	MOVQ            168(AX), R11
	MOVQ            192(AX), R12
	MOVQ            216(AX), AX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            (R13), R14
	MOVQ            24(R13), R15
	MOVQ            48(R13), R13
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_10x3_64_loop:
	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z29

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU64      (R11), Z30
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU64      (R12), Z30
	ADDQ           $0x40, R12
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 3 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 3 outputs
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R13)
	ADDQ      $0x40, R13

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_10x3_64_loop
	VZEROUPPER

mulGFNI_10x3_64_end:
	RET

// func mulAvxGFNI_10x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x3(SB), $8-88
	// Loading 11 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x3_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), R10
	MOVQ         168(AX), R11
	MOVQ         192(AX), R12
	MOVQ         216(AX), AX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R14
	MOVQ         24(R13), R15
	MOVQ         48(R13), R13
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_10x3_loop:
	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 3 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 3 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 3 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_10x3_loop
	VZEROUPPER

mulAvxGFNI_10x3_end:
	RET

// func mulGFNI_10x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x3_64Xor(SB), $8-88
	// Loading 27 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x3_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	VBROADCASTF32X2 208(CX), Z26
	MOVQ            in_base+24(FP), AX
	MOVQ            (AX), DX
	MOVQ            24(AX), BX
	MOVQ            48(AX), SI
	MOVQ            72(AX), DI
	MOVQ            96(AX), R8
	MOVQ            120(AX), R9
	MOVQ            144(AX), R10
	MOVQ            168(AX), R11
	MOVQ            192(AX), R12
	MOVQ            216(AX), AX
	MOVQ            out_base+48(FP), R13
	MOVQ            out_base+48(FP), R13
	MOVQ            (R13), R14
	MOVQ            24(R13), R15
	MOVQ            48(R13), R13
	MOVQ            start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulGFNI_10x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU64 (R14), Z27
	VMOVDQU64 (R15), Z28
	VMOVDQU64 (R13), Z29

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU64      (DX), Z30
	ADDQ           $0x40, DX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU64      (R11), Z30
	ADDQ           $0x40, R11
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU64      (R12), Z30
	ADDQ           $0x40, R12
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 3 outputs
	VMOVDQU64           (AX), Z30
	ADDQ                $0x40, AX
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 3 outputs
	VMOVDQU64 Z27, (R14)
	ADDQ      $0x40, R14
	VMOVDQU64 Z28, (R15)
	ADDQ      $0x40, R15
	VMOVDQU64 Z29, (R13)
	ADDQ      $0x40, R13

	// Prepare for next loop
	DECQ BP
	JNZ  mulGFNI_10x3_64Xor_loop
	VZEROUPPER

mulGFNI_10x3_64Xor_end:
	RET

// func mulAvxGFNI_10x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x3Xor(SB), $8-88
	// Loading 11 of 30 tables to registers
	// Destination kept in GP registers
	// Full registers estimated 35 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x3Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	VBROADCASTSD 80(CX), Y10
	MOVQ         in_base+24(FP), AX
	MOVQ         (AX), DX
	MOVQ         24(AX), BX
	MOVQ         48(AX), SI
	MOVQ         72(AX), DI
	MOVQ         96(AX), R8
	MOVQ         120(AX), R9
	MOVQ         144(AX), R10
	MOVQ         168(AX), R11
	MOVQ         192(AX), R12
	MOVQ         216(AX), AX
	MOVQ         out_base+48(FP), R13
	MOVQ         out_base+48(FP), R13
	MOVQ         (R13), R14
	MOVQ         24(R13), R15
	MOVQ         48(R13), R13
	MOVQ         start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ BP, DX
	ADDQ BP, BX
	ADDQ BP, SI
	ADDQ BP, DI
	ADDQ BP, R8
	ADDQ BP, R9
	ADDQ BP, R10
	ADDQ BP, R11
	ADDQ BP, R12
	ADDQ BP, AX

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x05, BP

mulAvxGFNI_10x3Xor_loop:
	// Load 3 outputs
	VMOVDQU (R14), Y11
	VMOVDQU (R15), Y12
	VMOVDQU (R13), Y13

	// Load and process 32 bytes from input 0 to 3 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 3 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 3 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 3 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 3 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 3 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 3 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 3 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 3 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 3 outputs
	VMOVDQU        (AX), Y14
	ADDQ           $0x20, AX
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 3 outputs
	VMOVDQU Y11, (R14)
	ADDQ    $0x20, R14
	VMOVDQU Y12, (R15)
	ADDQ    $0x20, R15
	VMOVDQU Y13, (R13)
	ADDQ    $0x20, R13

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxGFNI_10x3Xor_loop
	VZEROUPPER

mulAvxGFNI_10x3Xor_end:
	RET

// func mulAvxTwo_10x3_64Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x3_64Xor(SB), $8-88
	// Loading no tables to registers
	// Destination kept in GP registers
	// Full registers estimated 130 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x06, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x3_64Xor_end
	MOVQ  in_base+24(FP), AX
	MOVQ  (AX), DX
	MOVQ  24(AX), BX
	MOVQ  48(AX), SI
	MOVQ  72(AX), DI
	MOVQ  96(AX), R8
	MOVQ  120(AX), R9
	MOVQ  144(AX), R10
	MOVQ  168(AX), R11
	MOVQ  192(AX), R12
	MOVQ  216(AX), AX
	MOVQ  out_base+48(FP), R13
	MOVQ  (R13), R14
	MOVQ  24(R13), R15
	MOVQ  48(R13), R13
	MOVQ  start+72(FP), BP

	// Add start offset to output
	ADDQ BP, R14
	ADDQ BP, R15
	ADDQ BP, R13

	// Add start offset to input
	ADDQ         BP, DX
	ADDQ         BP, BX
	ADDQ         BP, SI
	ADDQ         BP, DI
	ADDQ         BP, R8
	ADDQ         BP, R9
	ADDQ         BP, R10
	ADDQ         BP, R11
	ADDQ         BP, R12
	ADDQ         BP, AX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

	// Reload length to save a register
	MOVQ n+80(FP), BP
	SHRQ $0x06, BP

mulAvxTwo_10x3_64Xor_loop:
	// Load 3 outputs
	VMOVDQU (R14), Y0
	VMOVDQU 32(R14), Y1
	VMOVDQU (R15), Y2
	VMOVDQU 32(R15), Y3
	VMOVDQU (R13), Y4
	VMOVDQU 32(R13), Y5

	// Load and process 64 bytes from input 0 to 3 outputs
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y13
	ADDQ    $0x40, DX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 1 to 3 outputs
	VMOVDQU (BX), Y11
	VMOVDQU 32(BX), Y13
	ADDQ    $0x40, BX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 2 to 3 outputs
	VMOVDQU (SI), Y11
	VMOVDQU 32(SI), Y13
	ADDQ    $0x40, SI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 3 to 3 outputs
	VMOVDQU (DI), Y11
	VMOVDQU 32(DI), Y13
	ADDQ    $0x40, DI
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 4 to 3 outputs
	VMOVDQU (R8), Y11
	VMOVDQU 32(R8), Y13
	ADDQ    $0x40, R8
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 5 to 3 outputs
	VMOVDQU (R9), Y11
	VMOVDQU 32(R9), Y13
	ADDQ    $0x40, R9
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 6 to 3 outputs
	VMOVDQU (R10), Y11
	VMOVDQU 32(R10), Y13
	ADDQ    $0x40, R10
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 7 to 3 outputs
	VMOVDQU (R11), Y11
	VMOVDQU 32(R11), Y13
	ADDQ    $0x40, R11
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 8 to 3 outputs
	VMOVDQU (R12), Y11
	VMOVDQU 32(R12), Y13
	ADDQ    $0x40, R12
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Load and process 64 bytes from input 9 to 3 outputs
	VMOVDQU (AX), Y11
	VMOVDQU 32(AX), Y13
	ADDQ    $0x40, AX
	VPSRLQ  $0x04, Y11, Y12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y6, Y11, Y11
	VPAND   Y6, Y13, Y13
	VPAND   Y6, Y12, Y12
	VPAND   Y6, Y14, Y14
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y13, Y7, Y9
	VPSHUFB Y11, Y7, Y7
	VPSHUFB Y14, Y8, Y10
	VPSHUFB Y12, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// Store 3 outputs
	VMOVDQU Y0, (R14)
	VMOVDQU Y1, 32(R14)
	ADDQ    $0x40, R14
	VMOVDQU Y2, (R15)
	VMOVDQU Y3, 32(R15)
	ADDQ    $0x40, R15
	VMOVDQU Y4, (R13)
	VMOVDQU Y5, 32(R13)
	ADDQ    $0x40, R13

	// Prepare for next loop
	DECQ BP
	JNZ  mulAvxTwo_10x3_64Xor_loop
	VZEROUPPER

mulAvxTwo_10x3_64Xor_end:
	RET

// func mulAvxTwo_10x4(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x4(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 89 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x4_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_10x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y0
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y1
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y2
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	VPXOR   Y5, Y6, Y3

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (R11), Y7
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU (R12), Y7
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1792(CX), Y5
	VMOVDQU 1824(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1856(CX), Y5
	VMOVDQU 1888(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1920(CX), Y5
	VMOVDQU 1952(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1984(CX), Y5
	VMOVDQU 2016(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU (R13), Y7
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 2048(CX), Y5
	VMOVDQU 2080(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 2112(CX), Y5
	VMOVDQU 2144(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 2176(CX), Y5
	VMOVDQU 2208(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 2240(CX), Y5
	VMOVDQU 2272(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 9 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 2304(CX), Y5
	VMOVDQU 2336(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 2368(CX), Y5
	VMOVDQU 2400(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 2432(CX), Y5
	VMOVDQU 2464(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 2496(CX), Y5
	VMOVDQU 2528(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x4_loop
	VZEROUPPER

mulAvxTwo_10x4_end:
	RET

// func mulGFNI_10x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x4_64(SB), $8-88
	// Loading 26 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 46 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x4_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x4_64_loop:
	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 4 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 4 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 4 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x4_64_loop
	VZEROUPPER

mulGFNI_10x4_64_end:
	RET

// func mulAvxGFNI_10x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x4(SB), $8-88
	// Loading 10 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 46 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x4_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x4_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	MOVQ    (R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x4_loop
	VZEROUPPER

mulAvxGFNI_10x4_end:
	RET

// func mulGFNI_10x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x4_64Xor(SB), $8-88
	// Loading 26 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 46 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x4_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	VBROADCASTF32X2 200(CX), Z25
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x4_64Xor_loop:
	// Load 4 outputs
	MOVQ      (R14), BP
	VMOVDQU64 (BP)(R15*1), Z26
	MOVQ      24(R14), BP
	VMOVDQU64 (BP)(R15*1), Z27
	MOVQ      48(R14), BP
	VMOVDQU64 (BP)(R15*1), Z28
	MOVQ      72(R14), BP
	VMOVDQU64 (BP)(R15*1), Z29

	// Load and process 64 bytes from input 0 to 4 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 4 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 4 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 4 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 4 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 4 outputs
	VMOVDQU64      (R10), Z30
	ADDQ           $0x40, R10
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 4 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB      $0x00, Z24, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z25, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 4 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 4 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 4 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 4 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x4_64Xor_loop
	VZEROUPPER

mulGFNI_10x4_64Xor_end:
	RET

// func mulAvxGFNI_10x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x4Xor(SB), $8-88
	// Loading 10 of 40 tables to registers
	// Destination kept on stack
	// Full registers estimated 46 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x4Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	VBROADCASTSD 72(CX), Y9
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x4Xor_loop:
	// Load 4 outputs
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y10
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y11
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y12
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y13

	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 4 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 4 outputs
	MOVQ    (R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x4Xor_loop
	VZEROUPPER

mulAvxGFNI_10x4Xor_end:
	RET

// func mulAvxTwo_10x4Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x4Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 89 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x4Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X4
	VPBROADCASTB X4, Y4

mulAvxTwo_10x4Xor_loop:
	// Load and process 32 bytes from input 0 to 4 outputs
	VMOVDQU (BX), Y7
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y0
	VMOVDQU (CX), Y5
	VMOVDQU 32(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y1
	VMOVDQU 64(CX), Y5
	VMOVDQU 96(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y2
	VMOVDQU 128(CX), Y5
	VMOVDQU 160(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y3
	VMOVDQU 192(CX), Y5
	VMOVDQU 224(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 1 to 4 outputs
	VMOVDQU (SI), Y7
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 256(CX), Y5
	VMOVDQU 288(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 320(CX), Y5
	VMOVDQU 352(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 384(CX), Y5
	VMOVDQU 416(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 448(CX), Y5
	VMOVDQU 480(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 2 to 4 outputs
	VMOVDQU (DI), Y7
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 512(CX), Y5
	VMOVDQU 544(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 576(CX), Y5
	VMOVDQU 608(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 640(CX), Y5
	VMOVDQU 672(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 704(CX), Y5
	VMOVDQU 736(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 3 to 4 outputs
	VMOVDQU (R8), Y7
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 768(CX), Y5
	VMOVDQU 800(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 832(CX), Y5
	VMOVDQU 864(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 896(CX), Y5
	VMOVDQU 928(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 960(CX), Y5
	VMOVDQU 992(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 4 to 4 outputs
	VMOVDQU (R9), Y7
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1024(CX), Y5
	VMOVDQU 1056(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1088(CX), Y5
	VMOVDQU 1120(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1152(CX), Y5
	VMOVDQU 1184(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1216(CX), Y5
	VMOVDQU 1248(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 5 to 4 outputs
	VMOVDQU (R10), Y7
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1280(CX), Y5
	VMOVDQU 1312(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1344(CX), Y5
	VMOVDQU 1376(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1408(CX), Y5
	VMOVDQU 1440(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1472(CX), Y5
	VMOVDQU 1504(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 6 to 4 outputs
	VMOVDQU (R11), Y7
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1536(CX), Y5
	VMOVDQU 1568(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1600(CX), Y5
	VMOVDQU 1632(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1664(CX), Y5
	VMOVDQU 1696(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1728(CX), Y5
	VMOVDQU 1760(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 7 to 4 outputs
	VMOVDQU (R12), Y7
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 1792(CX), Y5
	VMOVDQU 1824(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 1856(CX), Y5
	VMOVDQU 1888(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 1920(CX), Y5
	VMOVDQU 1952(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 1984(CX), Y5
	VMOVDQU 2016(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 8 to 4 outputs
	VMOVDQU (R13), Y7
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 2048(CX), Y5
	VMOVDQU 2080(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 2112(CX), Y5
	VMOVDQU 2144(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 2176(CX), Y5
	VMOVDQU 2208(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 2240(CX), Y5
	VMOVDQU 2272(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Load and process 32 bytes from input 9 to 4 outputs
	VMOVDQU (DX), Y7
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y7, Y8
	VPAND   Y4, Y7, Y7
	VPAND   Y4, Y8, Y8
	VMOVDQU 2304(CX), Y5
	VMOVDQU 2336(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y0)
	VMOVDQU 2368(CX), Y5
	VMOVDQU 2400(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y1)
	VMOVDQU 2432(CX), Y5
	VMOVDQU 2464(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y2)
	VMOVDQU 2496(CX), Y5
	VMOVDQU 2528(CX), Y6
	VPSHUFB Y7, Y5, Y5
	VPSHUFB Y8, Y6, Y6
	XOR3WAY( $0x00, Y5, Y6, Y3)

	// Store 4 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x4Xor_loop
	VZEROUPPER

mulAvxTwo_10x4Xor_end:
	RET

// func mulAvxTwo_10x5(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x5(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 110 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x5_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_10x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y0
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y1
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y2
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y3
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	VPXOR   Y6, Y7, Y4

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (R11), Y8
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU (R12), Y8
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2240(CX), Y6
	VMOVDQU 2272(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2304(CX), Y6
	VMOVDQU 2336(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2368(CX), Y6
	VMOVDQU 2400(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2432(CX), Y6
	VMOVDQU 2464(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2496(CX), Y6
	VMOVDQU 2528(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU (R13), Y8
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2560(CX), Y6
	VMOVDQU 2592(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2624(CX), Y6
	VMOVDQU 2656(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2688(CX), Y6
	VMOVDQU 2720(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2752(CX), Y6
	VMOVDQU 2784(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2816(CX), Y6
	VMOVDQU 2848(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 9 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2880(CX), Y6
	VMOVDQU 2912(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2944(CX), Y6
	VMOVDQU 2976(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 3008(CX), Y6
	VMOVDQU 3040(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 3072(CX), Y6
	VMOVDQU 3104(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 3136(CX), Y6
	VMOVDQU 3168(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x5_loop
	VZEROUPPER

mulAvxTwo_10x5_end:
	RET

// func mulGFNI_10x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x5_64(SB), $8-88
	// Loading 25 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 57 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x5_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x5_64_loop:
	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 5 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 5 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x5_64_loop
	VZEROUPPER

mulGFNI_10x5_64_end:
	RET

// func mulAvxGFNI_10x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x5(SB), $8-88
	// Loading 9 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 57 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x5_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x5_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	MOVQ    (R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x5_loop
	VZEROUPPER

mulAvxGFNI_10x5_end:
	RET

// func mulGFNI_10x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x5_64Xor(SB), $8-88
	// Loading 25 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 57 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x5_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	VBROADCASTF32X2 192(CX), Z24
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x5_64Xor_loop:
	// Load 5 outputs
	MOVQ      (R14), BP
	VMOVDQU64 (BP)(R15*1), Z25
	MOVQ      24(R14), BP
	VMOVDQU64 (BP)(R15*1), Z26
	MOVQ      48(R14), BP
	VMOVDQU64 (BP)(R15*1), Z27
	MOVQ      72(R14), BP
	VMOVDQU64 (BP)(R15*1), Z28
	MOVQ      96(R14), BP
	VMOVDQU64 (BP)(R15*1), Z29

	// Load and process 64 bytes from input 0 to 5 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 5 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 5 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 5 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 5 outputs
	VMOVDQU64      (R9), Z30
	ADDQ           $0x40, R9
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 5 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 5 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 5 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 5 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 5 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 5 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x5_64Xor_loop
	VZEROUPPER

mulGFNI_10x5_64Xor_end:
	RET

// func mulAvxGFNI_10x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x5Xor(SB), $8-88
	// Loading 9 of 50 tables to registers
	// Destination kept on stack
	// Full registers estimated 57 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x5Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	VBROADCASTSD 64(CX), Y8
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x5Xor_loop:
	// Load 5 outputs
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y9
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y10
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y11
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y12
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y13

	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 5 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 5 outputs
	MOVQ    (R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x5Xor_loop
	VZEROUPPER

mulAvxGFNI_10x5Xor_end:
	RET

// func mulAvxTwo_10x5Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x5Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 110 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x5Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X5
	VPBROADCASTB X5, Y5

mulAvxTwo_10x5Xor_loop:
	// Load and process 32 bytes from input 0 to 5 outputs
	VMOVDQU (BX), Y8
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y0
	VMOVDQU (CX), Y6
	VMOVDQU 32(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y1
	VMOVDQU 64(CX), Y6
	VMOVDQU 96(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y2
	VMOVDQU 128(CX), Y6
	VMOVDQU 160(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y3
	VMOVDQU 192(CX), Y6
	VMOVDQU 224(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y4
	VMOVDQU 256(CX), Y6
	VMOVDQU 288(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 1 to 5 outputs
	VMOVDQU (SI), Y8
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 320(CX), Y6
	VMOVDQU 352(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 384(CX), Y6
	VMOVDQU 416(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 448(CX), Y6
	VMOVDQU 480(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 512(CX), Y6
	VMOVDQU 544(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 576(CX), Y6
	VMOVDQU 608(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 2 to 5 outputs
	VMOVDQU (DI), Y8
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 640(CX), Y6
	VMOVDQU 672(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 704(CX), Y6
	VMOVDQU 736(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 768(CX), Y6
	VMOVDQU 800(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 832(CX), Y6
	VMOVDQU 864(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 896(CX), Y6
	VMOVDQU 928(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 3 to 5 outputs
	VMOVDQU (R8), Y8
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 960(CX), Y6
	VMOVDQU 992(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1024(CX), Y6
	VMOVDQU 1056(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1088(CX), Y6
	VMOVDQU 1120(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1152(CX), Y6
	VMOVDQU 1184(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1216(CX), Y6
	VMOVDQU 1248(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 4 to 5 outputs
	VMOVDQU (R9), Y8
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1280(CX), Y6
	VMOVDQU 1312(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1344(CX), Y6
	VMOVDQU 1376(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1408(CX), Y6
	VMOVDQU 1440(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1472(CX), Y6
	VMOVDQU 1504(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1536(CX), Y6
	VMOVDQU 1568(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 5 to 5 outputs
	VMOVDQU (R10), Y8
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1600(CX), Y6
	VMOVDQU 1632(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1664(CX), Y6
	VMOVDQU 1696(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 1728(CX), Y6
	VMOVDQU 1760(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 1792(CX), Y6
	VMOVDQU 1824(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 1856(CX), Y6
	VMOVDQU 1888(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 6 to 5 outputs
	VMOVDQU (R11), Y8
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 1920(CX), Y6
	VMOVDQU 1952(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 1984(CX), Y6
	VMOVDQU 2016(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2048(CX), Y6
	VMOVDQU 2080(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2112(CX), Y6
	VMOVDQU 2144(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2176(CX), Y6
	VMOVDQU 2208(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 7 to 5 outputs
	VMOVDQU (R12), Y8
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2240(CX), Y6
	VMOVDQU 2272(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2304(CX), Y6
	VMOVDQU 2336(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2368(CX), Y6
	VMOVDQU 2400(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2432(CX), Y6
	VMOVDQU 2464(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2496(CX), Y6
	VMOVDQU 2528(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 8 to 5 outputs
	VMOVDQU (R13), Y8
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2560(CX), Y6
	VMOVDQU 2592(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2624(CX), Y6
	VMOVDQU 2656(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 2688(CX), Y6
	VMOVDQU 2720(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 2752(CX), Y6
	VMOVDQU 2784(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 2816(CX), Y6
	VMOVDQU 2848(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Load and process 32 bytes from input 9 to 5 outputs
	VMOVDQU (DX), Y8
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y8, Y9
	VPAND   Y5, Y8, Y8
	VPAND   Y5, Y9, Y9
	VMOVDQU 2880(CX), Y6
	VMOVDQU 2912(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y0)
	VMOVDQU 2944(CX), Y6
	VMOVDQU 2976(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y1)
	VMOVDQU 3008(CX), Y6
	VMOVDQU 3040(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y2)
	VMOVDQU 3072(CX), Y6
	VMOVDQU 3104(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y3)
	VMOVDQU 3136(CX), Y6
	VMOVDQU 3168(CX), Y7
	VPSHUFB Y8, Y6, Y6
	VPSHUFB Y9, Y7, Y7
	XOR3WAY( $0x00, Y6, Y7, Y4)

	// Store 5 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x5Xor_loop
	VZEROUPPER

mulAvxTwo_10x5Xor_end:
	RET

// func mulAvxTwo_10x6(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x6(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 131 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x6_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_10x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y0
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y1
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y2
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y3
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y4
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	VPXOR   Y7, Y8, Y5

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R10), Y9
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (R11), Y9
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU (R12), Y9
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2688(CX), Y7
	VMOVDQU 2720(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2752(CX), Y7
	VMOVDQU 2784(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2816(CX), Y7
	VMOVDQU 2848(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2880(CX), Y7
	VMOVDQU 2912(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2944(CX), Y7
	VMOVDQU 2976(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3008(CX), Y7
	VMOVDQU 3040(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU (R13), Y9
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 3072(CX), Y7
	VMOVDQU 3104(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 3136(CX), Y7
	VMOVDQU 3168(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 3200(CX), Y7
	VMOVDQU 3232(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 3264(CX), Y7
	VMOVDQU 3296(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 3328(CX), Y7
	VMOVDQU 3360(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3392(CX), Y7
	VMOVDQU 3424(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 9 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 3456(CX), Y7
	VMOVDQU 3488(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 3520(CX), Y7
	VMOVDQU 3552(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 3584(CX), Y7
	VMOVDQU 3616(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 3648(CX), Y7
	VMOVDQU 3680(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 3712(CX), Y7
	VMOVDQU 3744(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3776(CX), Y7
	VMOVDQU 3808(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x6_loop
	VZEROUPPER

mulAvxTwo_10x6_end:
	RET

// func mulGFNI_10x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x6_64(SB), $8-88
	// Loading 24 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 68 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x6_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x6_64_loop:
	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 6 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 6 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x6_64_loop
	VZEROUPPER

mulGFNI_10x6_64_end:
	RET

// func mulAvxGFNI_10x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x6(SB), $8-88
	// Loading 8 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 68 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x6_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x6_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	MOVQ    (R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x6_loop
	VZEROUPPER

mulAvxGFNI_10x6_end:
	RET

// func mulGFNI_10x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x6_64Xor(SB), $8-88
	// Loading 24 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 68 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x6_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	VBROADCASTF32X2 184(CX), Z23
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x6_64Xor_loop:
	// Load 6 outputs
	MOVQ      (R14), BP
	VMOVDQU64 (BP)(R15*1), Z24
	MOVQ      24(R14), BP
	VMOVDQU64 (BP)(R15*1), Z25
	MOVQ      48(R14), BP
	VMOVDQU64 (BP)(R15*1), Z26
	MOVQ      72(R14), BP
	VMOVDQU64 (BP)(R15*1), Z27
	MOVQ      96(R14), BP
	VMOVDQU64 (BP)(R15*1), Z28
	MOVQ      120(R14), BP
	VMOVDQU64 (BP)(R15*1), Z29

	// Load and process 64 bytes from input 0 to 6 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 6 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 6 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 6 outputs
	VMOVDQU64      (R8), Z30
	ADDQ           $0x40, R8
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 6 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 6 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 6 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 6 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 6 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 6 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 6 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x6_64Xor_loop
	VZEROUPPER

mulGFNI_10x6_64Xor_end:
	RET

// func mulAvxGFNI_10x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x6Xor(SB), $8-88
	// Loading 8 of 60 tables to registers
	// Destination kept on stack
	// Full registers estimated 68 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x6Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	VBROADCASTSD 56(CX), Y7
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x6Xor_loop:
	// Load 6 outputs
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y8
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y9
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y10
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y11
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y12
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y13

	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 6 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 6 outputs
	MOVQ    (R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x6Xor_loop
	VZEROUPPER

mulAvxGFNI_10x6Xor_end:
	RET

// func mulAvxTwo_10x6Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x6Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 131 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x6Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X6
	VPBROADCASTB X6, Y6

mulAvxTwo_10x6Xor_loop:
	// Load and process 32 bytes from input 0 to 6 outputs
	VMOVDQU (BX), Y9
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y0
	VMOVDQU (CX), Y7
	VMOVDQU 32(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y1
	VMOVDQU 64(CX), Y7
	VMOVDQU 96(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y2
	VMOVDQU 128(CX), Y7
	VMOVDQU 160(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y3
	VMOVDQU 192(CX), Y7
	VMOVDQU 224(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y4
	VMOVDQU 256(CX), Y7
	VMOVDQU 288(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y5
	VMOVDQU 320(CX), Y7
	VMOVDQU 352(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 1 to 6 outputs
	VMOVDQU (SI), Y9
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 384(CX), Y7
	VMOVDQU 416(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 448(CX), Y7
	VMOVDQU 480(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 512(CX), Y7
	VMOVDQU 544(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 576(CX), Y7
	VMOVDQU 608(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 640(CX), Y7
	VMOVDQU 672(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 704(CX), Y7
	VMOVDQU 736(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 2 to 6 outputs
	VMOVDQU (DI), Y9
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 768(CX), Y7
	VMOVDQU 800(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 832(CX), Y7
	VMOVDQU 864(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 896(CX), Y7
	VMOVDQU 928(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 960(CX), Y7
	VMOVDQU 992(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1024(CX), Y7
	VMOVDQU 1056(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1088(CX), Y7
	VMOVDQU 1120(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 3 to 6 outputs
	VMOVDQU (R8), Y9
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1152(CX), Y7
	VMOVDQU 1184(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1216(CX), Y7
	VMOVDQU 1248(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1280(CX), Y7
	VMOVDQU 1312(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1344(CX), Y7
	VMOVDQU 1376(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1408(CX), Y7
	VMOVDQU 1440(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1472(CX), Y7
	VMOVDQU 1504(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 4 to 6 outputs
	VMOVDQU (R9), Y9
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1536(CX), Y7
	VMOVDQU 1568(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1600(CX), Y7
	VMOVDQU 1632(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 1664(CX), Y7
	VMOVDQU 1696(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 1728(CX), Y7
	VMOVDQU 1760(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 1792(CX), Y7
	VMOVDQU 1824(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 1856(CX), Y7
	VMOVDQU 1888(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 5 to 6 outputs
	VMOVDQU (R10), Y9
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 1920(CX), Y7
	VMOVDQU 1952(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 1984(CX), Y7
	VMOVDQU 2016(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2048(CX), Y7
	VMOVDQU 2080(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2112(CX), Y7
	VMOVDQU 2144(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2176(CX), Y7
	VMOVDQU 2208(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2240(CX), Y7
	VMOVDQU 2272(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 6 to 6 outputs
	VMOVDQU (R11), Y9
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2304(CX), Y7
	VMOVDQU 2336(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2368(CX), Y7
	VMOVDQU 2400(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2432(CX), Y7
	VMOVDQU 2464(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2496(CX), Y7
	VMOVDQU 2528(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2560(CX), Y7
	VMOVDQU 2592(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 2624(CX), Y7
	VMOVDQU 2656(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 7 to 6 outputs
	VMOVDQU (R12), Y9
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 2688(CX), Y7
	VMOVDQU 2720(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 2752(CX), Y7
	VMOVDQU 2784(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 2816(CX), Y7
	VMOVDQU 2848(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 2880(CX), Y7
	VMOVDQU 2912(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 2944(CX), Y7
	VMOVDQU 2976(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3008(CX), Y7
	VMOVDQU 3040(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 8 to 6 outputs
	VMOVDQU (R13), Y9
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 3072(CX), Y7
	VMOVDQU 3104(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 3136(CX), Y7
	VMOVDQU 3168(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 3200(CX), Y7
	VMOVDQU 3232(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 3264(CX), Y7
	VMOVDQU 3296(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 3328(CX), Y7
	VMOVDQU 3360(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3392(CX), Y7
	VMOVDQU 3424(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Load and process 32 bytes from input 9 to 6 outputs
	VMOVDQU (DX), Y9
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y9, Y10
	VPAND   Y6, Y9, Y9
	VPAND   Y6, Y10, Y10
	VMOVDQU 3456(CX), Y7
	VMOVDQU 3488(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y0)
	VMOVDQU 3520(CX), Y7
	VMOVDQU 3552(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y1)
	VMOVDQU 3584(CX), Y7
	VMOVDQU 3616(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y2)
	VMOVDQU 3648(CX), Y7
	VMOVDQU 3680(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)
	VMOVDQU 3712(CX), Y7
	VMOVDQU 3744(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU 3776(CX), Y7
	VMOVDQU 3808(CX), Y8
	VPSHUFB Y9, Y7, Y7
	VPSHUFB Y10, Y8, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)

	// Store 6 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x6Xor_loop
	VZEROUPPER

mulAvxTwo_10x6Xor_end:
	RET

// func mulAvxTwo_10x7(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x7(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 152 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x7_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_10x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y0
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y1
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y2
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y3
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y4
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y5
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	VPXOR   Y8, Y9, Y6

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (R11), Y10
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU (R12), Y10
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3136(CX), Y8
	VMOVDQU 3168(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3200(CX), Y8
	VMOVDQU 3232(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3264(CX), Y8
	VMOVDQU 3296(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3328(CX), Y8
	VMOVDQU 3360(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3392(CX), Y8
	VMOVDQU 3424(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3456(CX), Y8
	VMOVDQU 3488(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3520(CX), Y8
	VMOVDQU 3552(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU (R13), Y10
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3584(CX), Y8
	VMOVDQU 3616(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3648(CX), Y8
	VMOVDQU 3680(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3712(CX), Y8
	VMOVDQU 3744(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3776(CX), Y8
	VMOVDQU 3808(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3840(CX), Y8
	VMOVDQU 3872(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3904(CX), Y8
	VMOVDQU 3936(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3968(CX), Y8
	VMOVDQU 4000(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 9 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 4032(CX), Y8
	VMOVDQU 4064(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 4096(CX), Y8
	VMOVDQU 4128(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 4160(CX), Y8
	VMOVDQU 4192(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 4224(CX), Y8
	VMOVDQU 4256(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 4288(CX), Y8
	VMOVDQU 4320(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 4352(CX), Y8
	VMOVDQU 4384(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 4416(CX), Y8
	VMOVDQU 4448(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x7_loop
	VZEROUPPER

mulAvxTwo_10x7_end:
	RET

// func mulGFNI_10x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x7_64(SB), $8-88
	// Loading 23 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 79 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x7_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x7_64_loop:
	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 7 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 7 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x7_64_loop
	VZEROUPPER

mulGFNI_10x7_64_end:
	RET

// func mulAvxGFNI_10x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x7(SB), $8-88
	// Loading 7 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 79 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x7_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x7_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x7_loop
	VZEROUPPER

mulAvxGFNI_10x7_end:
	RET

// func mulGFNI_10x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x7_64Xor(SB), $8-88
	// Loading 23 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 79 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x7_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	VBROADCASTF32X2 176(CX), Z22
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x7_64Xor_loop:
	// Load 7 outputs
	MOVQ      (R14), BP
	VMOVDQU64 (BP)(R15*1), Z23
	MOVQ      24(R14), BP
	VMOVDQU64 (BP)(R15*1), Z24
	MOVQ      48(R14), BP
	VMOVDQU64 (BP)(R15*1), Z25
	MOVQ      72(R14), BP
	VMOVDQU64 (BP)(R15*1), Z26
	MOVQ      96(R14), BP
	VMOVDQU64 (BP)(R15*1), Z27
	MOVQ      120(R14), BP
	VMOVDQU64 (BP)(R15*1), Z28
	MOVQ      144(R14), BP
	VMOVDQU64 (BP)(R15*1), Z29

	// Load and process 64 bytes from input 0 to 7 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 7 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 7 outputs
	VMOVDQU64      (DI), Z30
	ADDQ           $0x40, DI
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 7 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z22, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 7 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 7 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 7 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 7 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 7 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 7 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 7 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x7_64Xor_loop
	VZEROUPPER

mulGFNI_10x7_64Xor_end:
	RET

// func mulAvxGFNI_10x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x7Xor(SB), $8-88
	// Loading 7 of 70 tables to registers
	// Destination kept on stack
	// Full registers estimated 79 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x7Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	VBROADCASTSD 48(CX), Y6
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x7Xor_loop:
	// Load 7 outputs
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y7
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y8
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y9
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y10
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y11
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y12
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y13

	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 7 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 7 outputs
	MOVQ    (R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x7Xor_loop
	VZEROUPPER

mulAvxGFNI_10x7Xor_end:
	RET

// func mulAvxTwo_10x7Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x7Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 152 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x7Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X7
	VPBROADCASTB X7, Y7

mulAvxTwo_10x7Xor_loop:
	// Load and process 32 bytes from input 0 to 7 outputs
	VMOVDQU (BX), Y10
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y0
	VMOVDQU (CX), Y8
	VMOVDQU 32(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y1
	VMOVDQU 64(CX), Y8
	VMOVDQU 96(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y2
	VMOVDQU 128(CX), Y8
	VMOVDQU 160(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y3
	VMOVDQU 192(CX), Y8
	VMOVDQU 224(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y4
	VMOVDQU 256(CX), Y8
	VMOVDQU 288(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y5
	VMOVDQU 320(CX), Y8
	VMOVDQU 352(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y6
	VMOVDQU 384(CX), Y8
	VMOVDQU 416(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 1 to 7 outputs
	VMOVDQU (SI), Y10
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 448(CX), Y8
	VMOVDQU 480(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 512(CX), Y8
	VMOVDQU 544(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 576(CX), Y8
	VMOVDQU 608(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 640(CX), Y8
	VMOVDQU 672(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 704(CX), Y8
	VMOVDQU 736(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 768(CX), Y8
	VMOVDQU 800(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 832(CX), Y8
	VMOVDQU 864(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 2 to 7 outputs
	VMOVDQU (DI), Y10
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 896(CX), Y8
	VMOVDQU 928(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 960(CX), Y8
	VMOVDQU 992(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1024(CX), Y8
	VMOVDQU 1056(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1088(CX), Y8
	VMOVDQU 1120(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1152(CX), Y8
	VMOVDQU 1184(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1216(CX), Y8
	VMOVDQU 1248(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1280(CX), Y8
	VMOVDQU 1312(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 3 to 7 outputs
	VMOVDQU (R8), Y10
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1344(CX), Y8
	VMOVDQU 1376(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1408(CX), Y8
	VMOVDQU 1440(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1472(CX), Y8
	VMOVDQU 1504(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1536(CX), Y8
	VMOVDQU 1568(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 1600(CX), Y8
	VMOVDQU 1632(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 1664(CX), Y8
	VMOVDQU 1696(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 1728(CX), Y8
	VMOVDQU 1760(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 4 to 7 outputs
	VMOVDQU (R9), Y10
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 1792(CX), Y8
	VMOVDQU 1824(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 1856(CX), Y8
	VMOVDQU 1888(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 1920(CX), Y8
	VMOVDQU 1952(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 1984(CX), Y8
	VMOVDQU 2016(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2048(CX), Y8
	VMOVDQU 2080(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2112(CX), Y8
	VMOVDQU 2144(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2176(CX), Y8
	VMOVDQU 2208(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 5 to 7 outputs
	VMOVDQU (R10), Y10
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2240(CX), Y8
	VMOVDQU 2272(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2304(CX), Y8
	VMOVDQU 2336(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2368(CX), Y8
	VMOVDQU 2400(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2432(CX), Y8
	VMOVDQU 2464(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2496(CX), Y8
	VMOVDQU 2528(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 2560(CX), Y8
	VMOVDQU 2592(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 2624(CX), Y8
	VMOVDQU 2656(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 6 to 7 outputs
	VMOVDQU (R11), Y10
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 2688(CX), Y8
	VMOVDQU 2720(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 2752(CX), Y8
	VMOVDQU 2784(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 2816(CX), Y8
	VMOVDQU 2848(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 2880(CX), Y8
	VMOVDQU 2912(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 2944(CX), Y8
	VMOVDQU 2976(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3008(CX), Y8
	VMOVDQU 3040(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3072(CX), Y8
	VMOVDQU 3104(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 7 to 7 outputs
	VMOVDQU (R12), Y10
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3136(CX), Y8
	VMOVDQU 3168(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3200(CX), Y8
	VMOVDQU 3232(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3264(CX), Y8
	VMOVDQU 3296(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3328(CX), Y8
	VMOVDQU 3360(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3392(CX), Y8
	VMOVDQU 3424(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3456(CX), Y8
	VMOVDQU 3488(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3520(CX), Y8
	VMOVDQU 3552(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 8 to 7 outputs
	VMOVDQU (R13), Y10
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 3584(CX), Y8
	VMOVDQU 3616(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 3648(CX), Y8
	VMOVDQU 3680(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 3712(CX), Y8
	VMOVDQU 3744(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 3776(CX), Y8
	VMOVDQU 3808(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 3840(CX), Y8
	VMOVDQU 3872(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 3904(CX), Y8
	VMOVDQU 3936(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 3968(CX), Y8
	VMOVDQU 4000(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Load and process 32 bytes from input 9 to 7 outputs
	VMOVDQU (DX), Y10
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y10, Y11
	VPAND   Y7, Y10, Y10
	VPAND   Y7, Y11, Y11
	VMOVDQU 4032(CX), Y8
	VMOVDQU 4064(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y0)
	VMOVDQU 4096(CX), Y8
	VMOVDQU 4128(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y1)
	VMOVDQU 4160(CX), Y8
	VMOVDQU 4192(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y2)
	VMOVDQU 4224(CX), Y8
	VMOVDQU 4256(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y3)
	VMOVDQU 4288(CX), Y8
	VMOVDQU 4320(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y4)
	VMOVDQU 4352(CX), Y8
	VMOVDQU 4384(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y5)
	VMOVDQU 4416(CX), Y8
	VMOVDQU 4448(CX), Y9
	VPSHUFB Y10, Y8, Y8
	VPSHUFB Y11, Y9, Y9
	XOR3WAY( $0x00, Y8, Y9, Y6)

	// Store 7 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x7Xor_loop
	VZEROUPPER

mulAvxTwo_10x7Xor_end:
	RET

// func mulAvxTwo_10x8(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x8(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 173 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x8_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_10x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y0
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y1
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y2
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y3
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y4
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y5
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y6
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	VPXOR   Y9, Y10, Y7

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (R11), Y11
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU (R12), Y11
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3584(CX), Y9
	VMOVDQU 3616(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3648(CX), Y9
	VMOVDQU 3680(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3712(CX), Y9
	VMOVDQU 3744(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3776(CX), Y9
	VMOVDQU 3808(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3840(CX), Y9
	VMOVDQU 3872(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3904(CX), Y9
	VMOVDQU 3936(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3968(CX), Y9
	VMOVDQU 4000(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4032(CX), Y9
	VMOVDQU 4064(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU (R13), Y11
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 4096(CX), Y9
	VMOVDQU 4128(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 4160(CX), Y9
	VMOVDQU 4192(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 4224(CX), Y9
	VMOVDQU 4256(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 4288(CX), Y9
	VMOVDQU 4320(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 4352(CX), Y9
	VMOVDQU 4384(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 4416(CX), Y9
	VMOVDQU 4448(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 4480(CX), Y9
	VMOVDQU 4512(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4544(CX), Y9
	VMOVDQU 4576(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 9 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 4608(CX), Y9
	VMOVDQU 4640(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 4672(CX), Y9
	VMOVDQU 4704(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 4736(CX), Y9
	VMOVDQU 4768(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 4800(CX), Y9
	VMOVDQU 4832(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 4864(CX), Y9
	VMOVDQU 4896(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 4928(CX), Y9
	VMOVDQU 4960(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 4992(CX), Y9
	VMOVDQU 5024(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 5056(CX), Y9
	VMOVDQU 5088(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y7, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x8_loop
	VZEROUPPER

mulAvxTwo_10x8_end:
	RET

// func mulGFNI_10x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x8_64(SB), $8-88
	// Loading 22 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 90 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x8_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x8_64_loop:
	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 8 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 8 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z22, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      168(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x8_64_loop
	VZEROUPPER

mulGFNI_10x8_64_end:
	RET

// func mulAvxGFNI_10x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x8(SB), $8-88
	// Loading 6 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 90 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x8_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x8_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
	VBROADCASTSD   48(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   56(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x8_loop
	VZEROUPPER

mulAvxGFNI_10x8_end:
	RET

// func mulGFNI_10x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x8_64Xor(SB), $8-88
	// Loading 22 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 90 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x8_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	VBROADCASTF32X2 168(CX), Z21
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x8_64Xor_loop:
	// Load 8 outputs
	MOVQ      (R14), BP
	VMOVDQU64 (BP)(R15*1), Z22
	MOVQ      24(R14), BP
	VMOVDQU64 (BP)(R15*1), Z23
	MOVQ      48(R14), BP
	VMOVDQU64 (BP)(R15*1), Z24
	MOVQ      72(R14), BP
	VMOVDQU64 (BP)(R15*1), Z25
	MOVQ      96(R14), BP
	VMOVDQU64 (BP)(R15*1), Z26
	MOVQ      120(R14), BP
	VMOVDQU64 (BP)(R15*1), Z27
	MOVQ      144(R14), BP
	VMOVDQU64 (BP)(R15*1), Z28
	MOVQ      168(R14), BP
	VMOVDQU64 (BP)(R15*1), Z29

	// Load and process 64 bytes from input 0 to 8 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 8 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 8 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z16, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z17, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB      $0x00, Z21, Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 8 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 8 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 8 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 8 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 8 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 8 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 8 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 8 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z22, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      168(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x8_64Xor_loop
	VZEROUPPER

mulGFNI_10x8_64Xor_end:
	RET

// func mulAvxGFNI_10x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x8Xor(SB), $8-88
	// Loading 6 of 80 tables to registers
	// Destination kept on stack
	// Full registers estimated 90 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x8Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	VBROADCASTSD 40(CX), Y5
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x8Xor_loop:
	// Load 8 outputs
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y6
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y7
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y8
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y9
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y10
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y11
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y12
	MOVQ    168(R14), BP
	VMOVDQU (BP)(R15*1), Y13

	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 8 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 8 outputs
	MOVQ    (R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x8Xor_loop
	VZEROUPPER

mulAvxGFNI_10x8Xor_end:
	RET

// func mulAvxTwo_10x8Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x8Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 173 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x8Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X8
	VPBROADCASTB X8, Y8

mulAvxTwo_10x8Xor_loop:
	// Load and process 32 bytes from input 0 to 8 outputs
	VMOVDQU (BX), Y11
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y0
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y1
	VMOVDQU 64(CX), Y9
	VMOVDQU 96(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y2
	VMOVDQU 128(CX), Y9
	VMOVDQU 160(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y3
	VMOVDQU 192(CX), Y9
	VMOVDQU 224(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y4
	VMOVDQU 256(CX), Y9
	VMOVDQU 288(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y5
	VMOVDQU 320(CX), Y9
	VMOVDQU 352(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y6
	VMOVDQU 384(CX), Y9
	VMOVDQU 416(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	MOVQ    168(R14), BP
	VMOVDQU (BP)(R15*1), Y7
	VMOVDQU 448(CX), Y9
	VMOVDQU 480(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 1 to 8 outputs
	VMOVDQU (SI), Y11
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 512(CX), Y9
	VMOVDQU 544(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 576(CX), Y9
	VMOVDQU 608(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 640(CX), Y9
	VMOVDQU 672(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 704(CX), Y9
	VMOVDQU 736(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 768(CX), Y9
	VMOVDQU 800(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 832(CX), Y9
	VMOVDQU 864(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 896(CX), Y9
	VMOVDQU 928(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 960(CX), Y9
	VMOVDQU 992(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 2 to 8 outputs
	VMOVDQU (DI), Y11
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1024(CX), Y9
	VMOVDQU 1056(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1088(CX), Y9
	VMOVDQU 1120(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1152(CX), Y9
	VMOVDQU 1184(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1216(CX), Y9
	VMOVDQU 1248(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1280(CX), Y9
	VMOVDQU 1312(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1344(CX), Y9
	VMOVDQU 1376(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1408(CX), Y9
	VMOVDQU 1440(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1472(CX), Y9
	VMOVDQU 1504(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 3 to 8 outputs
	VMOVDQU (R8), Y11
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 1536(CX), Y9
	VMOVDQU 1568(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 1600(CX), Y9
	VMOVDQU 1632(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 1664(CX), Y9
	VMOVDQU 1696(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 1728(CX), Y9
	VMOVDQU 1760(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 1792(CX), Y9
	VMOVDQU 1824(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 1856(CX), Y9
	VMOVDQU 1888(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 1920(CX), Y9
	VMOVDQU 1952(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 1984(CX), Y9
	VMOVDQU 2016(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 4 to 8 outputs
	VMOVDQU (R9), Y11
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2048(CX), Y9
	VMOVDQU 2080(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2112(CX), Y9
	VMOVDQU 2144(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2176(CX), Y9
	VMOVDQU 2208(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2240(CX), Y9
	VMOVDQU 2272(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2304(CX), Y9
	VMOVDQU 2336(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2368(CX), Y9
	VMOVDQU 2400(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2432(CX), Y9
	VMOVDQU 2464(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 2496(CX), Y9
	VMOVDQU 2528(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 5 to 8 outputs
	VMOVDQU (R10), Y11
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 2560(CX), Y9
	VMOVDQU 2592(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 2624(CX), Y9
	VMOVDQU 2656(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 2688(CX), Y9
	VMOVDQU 2720(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 2752(CX), Y9
	VMOVDQU 2784(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 2816(CX), Y9
	VMOVDQU 2848(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 2880(CX), Y9
	VMOVDQU 2912(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 2944(CX), Y9
	VMOVDQU 2976(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3008(CX), Y9
	VMOVDQU 3040(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 6 to 8 outputs
	VMOVDQU (R11), Y11
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3072(CX), Y9
	VMOVDQU 3104(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3136(CX), Y9
	VMOVDQU 3168(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3200(CX), Y9
	VMOVDQU 3232(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3264(CX), Y9
	VMOVDQU 3296(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3328(CX), Y9
	VMOVDQU 3360(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3392(CX), Y9
	VMOVDQU 3424(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3456(CX), Y9
	VMOVDQU 3488(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 3520(CX), Y9
	VMOVDQU 3552(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 7 to 8 outputs
	VMOVDQU (R12), Y11
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 3584(CX), Y9
	VMOVDQU 3616(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 3648(CX), Y9
	VMOVDQU 3680(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 3712(CX), Y9
	VMOVDQU 3744(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 3776(CX), Y9
	VMOVDQU 3808(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 3840(CX), Y9
	VMOVDQU 3872(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 3904(CX), Y9
	VMOVDQU 3936(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 3968(CX), Y9
	VMOVDQU 4000(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4032(CX), Y9
	VMOVDQU 4064(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 8 to 8 outputs
	VMOVDQU (R13), Y11
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 4096(CX), Y9
	VMOVDQU 4128(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 4160(CX), Y9
	VMOVDQU 4192(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 4224(CX), Y9
	VMOVDQU 4256(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 4288(CX), Y9
	VMOVDQU 4320(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 4352(CX), Y9
	VMOVDQU 4384(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 4416(CX), Y9
	VMOVDQU 4448(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 4480(CX), Y9
	VMOVDQU 4512(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 4544(CX), Y9
	VMOVDQU 4576(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Load and process 32 bytes from input 9 to 8 outputs
	VMOVDQU (DX), Y11
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y11, Y12
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y12, Y12
	VMOVDQU 4608(CX), Y9
	VMOVDQU 4640(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y0)
	VMOVDQU 4672(CX), Y9
	VMOVDQU 4704(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y1)
	VMOVDQU 4736(CX), Y9
	VMOVDQU 4768(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y2)
	VMOVDQU 4800(CX), Y9
	VMOVDQU 4832(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y3)
	VMOVDQU 4864(CX), Y9
	VMOVDQU 4896(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y4)
	VMOVDQU 4928(CX), Y9
	VMOVDQU 4960(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)
	VMOVDQU 4992(CX), Y9
	VMOVDQU 5024(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y6)
	VMOVDQU 5056(CX), Y9
	VMOVDQU 5088(CX), Y10
	VPSHUFB Y11, Y9, Y9
	VPSHUFB Y12, Y10, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)

	// Store 8 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y7, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x8Xor_loop
	VZEROUPPER

mulAvxTwo_10x8Xor_end:
	RET

// func mulAvxTwo_10x9(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x9(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 194 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x9_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_10x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y0
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y1
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y2
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y3
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y4
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y5
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y6
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y7
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	VPXOR   Y10, Y11, Y8

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (R11), Y12
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU (R12), Y12
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4032(CX), Y10
	VMOVDQU 4064(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4096(CX), Y10
	VMOVDQU 4128(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4160(CX), Y10
	VMOVDQU 4192(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4224(CX), Y10
	VMOVDQU 4256(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4288(CX), Y10
	VMOVDQU 4320(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4352(CX), Y10
	VMOVDQU 4384(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4416(CX), Y10
	VMOVDQU 4448(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 4480(CX), Y10
	VMOVDQU 4512(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 4544(CX), Y10
	VMOVDQU 4576(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU (R13), Y12
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4608(CX), Y10
	VMOVDQU 4640(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4672(CX), Y10
	VMOVDQU 4704(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4736(CX), Y10
	VMOVDQU 4768(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4800(CX), Y10
	VMOVDQU 4832(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4864(CX), Y10
	VMOVDQU 4896(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4928(CX), Y10
	VMOVDQU 4960(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4992(CX), Y10
	VMOVDQU 5024(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 5056(CX), Y10
	VMOVDQU 5088(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 5120(CX), Y10
	VMOVDQU 5152(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 9 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 5184(CX), Y10
	VMOVDQU 5216(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 5248(CX), Y10
	VMOVDQU 5280(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 5312(CX), Y10
	VMOVDQU 5344(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 5376(CX), Y10
	VMOVDQU 5408(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 5440(CX), Y10
	VMOVDQU 5472(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 5504(CX), Y10
	VMOVDQU 5536(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 5568(CX), Y10
	VMOVDQU 5600(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 5632(CX), Y10
	VMOVDQU 5664(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 5696(CX), Y10
	VMOVDQU 5728(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y8, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x9_loop
	VZEROUPPER

mulAvxTwo_10x9_end:
	RET

// func mulGFNI_10x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x9_64(SB), $8-88
	// Loading 21 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 101 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x9_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x9_64_loop:
	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 9 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 9 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z21, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z22, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      168(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      192(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x9_64_loop
	VZEROUPPER

mulGFNI_10x9_64_end:
	RET

// func mulAvxGFNI_10x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x9(SB), $8-88
	// Loading 5 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 101 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x9_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x9_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
	VBROADCASTSD   40(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   48(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   56(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   64(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   648(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   656(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   664(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   672(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   680(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   688(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   696(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   704(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   712(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x9_loop
	VZEROUPPER

mulAvxGFNI_10x9_end:
	RET

// func mulGFNI_10x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x9_64Xor(SB), $8-88
	// Loading 21 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 101 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x9_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	VBROADCASTF32X2 160(CX), Z20
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x9_64Xor_loop:
	// Load 9 outputs
	MOVQ      (R14), BP
	VMOVDQU64 (BP)(R15*1), Z21
	MOVQ      24(R14), BP
	VMOVDQU64 (BP)(R15*1), Z22
	MOVQ      48(R14), BP
	VMOVDQU64 (BP)(R15*1), Z23
	MOVQ      72(R14), BP
	VMOVDQU64 (BP)(R15*1), Z24
	MOVQ      96(R14), BP
	VMOVDQU64 (BP)(R15*1), Z25
	MOVQ      120(R14), BP
	VMOVDQU64 (BP)(R15*1), Z26
	MOVQ      144(R14), BP
	VMOVDQU64 (BP)(R15*1), Z27
	MOVQ      168(R14), BP
	VMOVDQU64 (BP)(R15*1), Z28
	MOVQ      192(R14), BP
	VMOVDQU64 (BP)(R15*1), Z29

	// Load and process 64 bytes from input 0 to 9 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 9 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 9 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB      $0x00, Z18, Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB      $0x00, Z19, Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB      $0x00, Z20, Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 9 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 9 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 9 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 9 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 9 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 9 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 9 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 9 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z21, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z22, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      168(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      192(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x9_64Xor_loop
	VZEROUPPER

mulGFNI_10x9_64Xor_end:
	RET

// func mulAvxGFNI_10x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x9Xor(SB), $8-88
	// Loading 5 of 90 tables to registers
	// Destination kept on stack
	// Full registers estimated 101 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x9Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	VBROADCASTSD 32(CX), Y4
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x9Xor_loop:
	// Load 9 outputs
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y5
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y6
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y7
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y8
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y9
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y10
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y11
	MOVQ    168(R14), BP
	VMOVDQU (BP)(R15*1), Y12
	MOVQ    192(R14), BP
	VMOVDQU (BP)(R15*1), Y13

	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 9 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   648(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   656(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   664(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   672(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   680(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   688(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   696(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   704(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   712(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 9 outputs
	MOVQ    (R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x9Xor_loop
	VZEROUPPER

mulAvxGFNI_10x9Xor_end:
	RET

// func mulAvxTwo_10x9Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x9Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 194 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x9Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X9
	VPBROADCASTB X9, Y9

mulAvxTwo_10x9Xor_loop:
	// Load and process 32 bytes from input 0 to 9 outputs
	VMOVDQU (BX), Y12
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y0
	VMOVDQU (CX), Y10
	VMOVDQU 32(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y1
	VMOVDQU 64(CX), Y10
	VMOVDQU 96(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y2
	VMOVDQU 128(CX), Y10
	VMOVDQU 160(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y3
	VMOVDQU 192(CX), Y10
	VMOVDQU 224(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y4
	VMOVDQU 256(CX), Y10
	VMOVDQU 288(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y5
	VMOVDQU 320(CX), Y10
	VMOVDQU 352(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y6
	VMOVDQU 384(CX), Y10
	VMOVDQU 416(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	MOVQ    168(R14), BP
	VMOVDQU (BP)(R15*1), Y7
	VMOVDQU 448(CX), Y10
	VMOVDQU 480(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	MOVQ    192(R14), BP
	VMOVDQU (BP)(R15*1), Y8
	VMOVDQU 512(CX), Y10
	VMOVDQU 544(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 1 to 9 outputs
	VMOVDQU (SI), Y12
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 576(CX), Y10
	VMOVDQU 608(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 640(CX), Y10
	VMOVDQU 672(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 704(CX), Y10
	VMOVDQU 736(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 768(CX), Y10
	VMOVDQU 800(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 832(CX), Y10
	VMOVDQU 864(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 896(CX), Y10
	VMOVDQU 928(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 960(CX), Y10
	VMOVDQU 992(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1024(CX), Y10
	VMOVDQU 1056(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1088(CX), Y10
	VMOVDQU 1120(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 2 to 9 outputs
	VMOVDQU (DI), Y12
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1152(CX), Y10
	VMOVDQU 1184(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1216(CX), Y10
	VMOVDQU 1248(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1280(CX), Y10
	VMOVDQU 1312(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1344(CX), Y10
	VMOVDQU 1376(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1408(CX), Y10
	VMOVDQU 1440(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 1472(CX), Y10
	VMOVDQU 1504(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 1536(CX), Y10
	VMOVDQU 1568(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 1600(CX), Y10
	VMOVDQU 1632(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 1664(CX), Y10
	VMOVDQU 1696(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 3 to 9 outputs
	VMOVDQU (R8), Y12
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 1728(CX), Y10
	VMOVDQU 1760(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 1792(CX), Y10
	VMOVDQU 1824(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 1856(CX), Y10
	VMOVDQU 1888(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 1920(CX), Y10
	VMOVDQU 1952(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 1984(CX), Y10
	VMOVDQU 2016(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2048(CX), Y10
	VMOVDQU 2080(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2112(CX), Y10
	VMOVDQU 2144(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2176(CX), Y10
	VMOVDQU 2208(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2240(CX), Y10
	VMOVDQU 2272(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 4 to 9 outputs
	VMOVDQU (R9), Y12
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2304(CX), Y10
	VMOVDQU 2336(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2368(CX), Y10
	VMOVDQU 2400(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 2432(CX), Y10
	VMOVDQU 2464(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 2496(CX), Y10
	VMOVDQU 2528(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 2560(CX), Y10
	VMOVDQU 2592(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 2624(CX), Y10
	VMOVDQU 2656(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 2688(CX), Y10
	VMOVDQU 2720(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 2752(CX), Y10
	VMOVDQU 2784(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 2816(CX), Y10
	VMOVDQU 2848(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 5 to 9 outputs
	VMOVDQU (R10), Y12
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 2880(CX), Y10
	VMOVDQU 2912(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 2944(CX), Y10
	VMOVDQU 2976(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3008(CX), Y10
	VMOVDQU 3040(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3072(CX), Y10
	VMOVDQU 3104(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3136(CX), Y10
	VMOVDQU 3168(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3200(CX), Y10
	VMOVDQU 3232(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3264(CX), Y10
	VMOVDQU 3296(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3328(CX), Y10
	VMOVDQU 3360(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3392(CX), Y10
	VMOVDQU 3424(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 6 to 9 outputs
	VMOVDQU (R11), Y12
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 3456(CX), Y10
	VMOVDQU 3488(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 3520(CX), Y10
	VMOVDQU 3552(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 3584(CX), Y10
	VMOVDQU 3616(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 3648(CX), Y10
	VMOVDQU 3680(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 3712(CX), Y10
	VMOVDQU 3744(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 3776(CX), Y10
	VMOVDQU 3808(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 3840(CX), Y10
	VMOVDQU 3872(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 3904(CX), Y10
	VMOVDQU 3936(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 3968(CX), Y10
	VMOVDQU 4000(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 7 to 9 outputs
	VMOVDQU (R12), Y12
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4032(CX), Y10
	VMOVDQU 4064(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4096(CX), Y10
	VMOVDQU 4128(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4160(CX), Y10
	VMOVDQU 4192(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4224(CX), Y10
	VMOVDQU 4256(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4288(CX), Y10
	VMOVDQU 4320(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4352(CX), Y10
	VMOVDQU 4384(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4416(CX), Y10
	VMOVDQU 4448(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 4480(CX), Y10
	VMOVDQU 4512(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 4544(CX), Y10
	VMOVDQU 4576(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 8 to 9 outputs
	VMOVDQU (R13), Y12
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 4608(CX), Y10
	VMOVDQU 4640(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 4672(CX), Y10
	VMOVDQU 4704(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 4736(CX), Y10
	VMOVDQU 4768(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 4800(CX), Y10
	VMOVDQU 4832(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 4864(CX), Y10
	VMOVDQU 4896(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 4928(CX), Y10
	VMOVDQU 4960(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 4992(CX), Y10
	VMOVDQU 5024(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 5056(CX), Y10
	VMOVDQU 5088(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 5120(CX), Y10
	VMOVDQU 5152(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Load and process 32 bytes from input 9 to 9 outputs
	VMOVDQU (DX), Y12
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y12, Y13
	VPAND   Y9, Y12, Y12
	VPAND   Y9, Y13, Y13
	VMOVDQU 5184(CX), Y10
	VMOVDQU 5216(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y0)
	VMOVDQU 5248(CX), Y10
	VMOVDQU 5280(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y1)
	VMOVDQU 5312(CX), Y10
	VMOVDQU 5344(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y2)
	VMOVDQU 5376(CX), Y10
	VMOVDQU 5408(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y3)
	VMOVDQU 5440(CX), Y10
	VMOVDQU 5472(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y4)
	VMOVDQU 5504(CX), Y10
	VMOVDQU 5536(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y5)
	VMOVDQU 5568(CX), Y10
	VMOVDQU 5600(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y6)
	VMOVDQU 5632(CX), Y10
	VMOVDQU 5664(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y7)
	VMOVDQU 5696(CX), Y10
	VMOVDQU 5728(CX), Y11
	VPSHUFB Y12, Y10, Y10
	VPSHUFB Y13, Y11, Y11
	XOR3WAY( $0x00, Y10, Y11, Y8)

	// Store 9 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y8, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x9Xor_loop
	VZEROUPPER

mulAvxTwo_10x9Xor_end:
	RET

// func mulAvxTwo_10x10(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x10(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 215 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x10_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_10x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y0
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y1
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y2
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y3
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y4
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y5
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y6
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y7
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y8
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	VPXOR   Y11, Y12, Y9

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (R11), Y13
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU (R12), Y13
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 4480(CX), Y11
	VMOVDQU 4512(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 4544(CX), Y11
	VMOVDQU 4576(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 4608(CX), Y11
	VMOVDQU 4640(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4672(CX), Y11
	VMOVDQU 4704(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4736(CX), Y11
	VMOVDQU 4768(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4800(CX), Y11
	VMOVDQU 4832(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4864(CX), Y11
	VMOVDQU 4896(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4928(CX), Y11
	VMOVDQU 4960(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4992(CX), Y11
	VMOVDQU 5024(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5056(CX), Y11
	VMOVDQU 5088(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU (R13), Y13
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 5120(CX), Y11
	VMOVDQU 5152(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 5184(CX), Y11
	VMOVDQU 5216(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 5248(CX), Y11
	VMOVDQU 5280(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 5312(CX), Y11
	VMOVDQU 5344(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 5376(CX), Y11
	VMOVDQU 5408(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 5440(CX), Y11
	VMOVDQU 5472(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 5504(CX), Y11
	VMOVDQU 5536(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 5568(CX), Y11
	VMOVDQU 5600(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 5632(CX), Y11
	VMOVDQU 5664(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5696(CX), Y11
	VMOVDQU 5728(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 9 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 5760(CX), Y11
	VMOVDQU 5792(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 5824(CX), Y11
	VMOVDQU 5856(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 5888(CX), Y11
	VMOVDQU 5920(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 5952(CX), Y11
	VMOVDQU 5984(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 6016(CX), Y11
	VMOVDQU 6048(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 6080(CX), Y11
	VMOVDQU 6112(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 6144(CX), Y11
	VMOVDQU 6176(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 6208(CX), Y11
	VMOVDQU 6240(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 6272(CX), Y11
	VMOVDQU 6304(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 6336(CX), Y11
	VMOVDQU 6368(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    216(R14), BP
	VMOVDQU Y9, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x10_loop
	VZEROUPPER

mulAvxTwo_10x10_end:
	RET

// func mulGFNI_10x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x10_64(SB), $8-88
	// Loading 20 of 100 tables to registers
	// Destination kept on stack
	// Full registers estimated 112 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x10_64_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x10_64_loop:
	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 10 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 10 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 720(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 728(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 736(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 744(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 752(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 760(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 768(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 776(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 784(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 792(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z20, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z21, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z22, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      168(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      192(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      216(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x10_64_loop
	VZEROUPPER

mulGFNI_10x10_64_end:
	RET

// func mulAvxGFNI_10x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x10(SB), $8-88
	// Loading 4 of 100 tables to registers
	// Destination kept on stack
	// Full registers estimated 112 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x10_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x10_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
	VBROADCASTSD   32(CX), Y8
	VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
	VBROADCASTSD   40(CX), Y9
	VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
	VBROADCASTSD   48(CX), Y10
	VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
	VBROADCASTSD   56(CX), Y11
	VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
	VBROADCASTSD   64(CX), Y12
	VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
	VBROADCASTSD   72(CX), Y13
	VGF2P8AFFINEQB $0x00, Y13, Y14, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   648(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   656(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   664(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   672(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   680(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   688(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   696(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   704(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   712(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   720(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   728(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   736(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   744(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   752(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   760(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   768(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   776(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   784(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   792(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    216(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x10_loop
	VZEROUPPER

mulAvxGFNI_10x10_end:
	RET

// func mulGFNI_10x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·mulGFNI_10x10_64Xor(SB), $8-88
	// Loading 20 of 100 tables to registers
	// Destination kept on stack
	// Full registers estimated 112 YMM used
	MOVQ            n+80(FP), AX
	MOVQ            matrix_base+0(FP), CX
	SHRQ            $0x06, AX
	TESTQ           AX, AX
	JZ              mulGFNI_10x10_64Xor_end
	VBROADCASTF32X2 (CX), Z0
	VBROADCASTF32X2 8(CX), Z1
	VBROADCASTF32X2 16(CX), Z2
	VBROADCASTF32X2 24(CX), Z3
	VBROADCASTF32X2 32(CX), Z4
	VBROADCASTF32X2 40(CX), Z5
	VBROADCASTF32X2 48(CX), Z6
	VBROADCASTF32X2 56(CX), Z7
	VBROADCASTF32X2 64(CX), Z8
	VBROADCASTF32X2 72(CX), Z9
	VBROADCASTF32X2 80(CX), Z10
	VBROADCASTF32X2 88(CX), Z11
	VBROADCASTF32X2 96(CX), Z12
	VBROADCASTF32X2 104(CX), Z13
	VBROADCASTF32X2 112(CX), Z14
	VBROADCASTF32X2 120(CX), Z15
	VBROADCASTF32X2 128(CX), Z16
	VBROADCASTF32X2 136(CX), Z17
	VBROADCASTF32X2 144(CX), Z18
	VBROADCASTF32X2 152(CX), Z19
	MOVQ            in_base+24(FP), DX
	MOVQ            (DX), BX
	MOVQ            24(DX), SI
	MOVQ            48(DX), DI
	MOVQ            72(DX), R8
	MOVQ            96(DX), R9
	MOVQ            120(DX), R10
	MOVQ            144(DX), R11
	MOVQ            168(DX), R12
	MOVQ            192(DX), R13
	MOVQ            216(DX), DX
	MOVQ            out_base+48(FP), R14
	MOVQ            out_base+48(FP), R14
	MOVQ            start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulGFNI_10x10_64Xor_loop:
	// Load 10 outputs
	MOVQ      (R14), BP
	VMOVDQU64 (BP)(R15*1), Z20
	MOVQ      24(R14), BP
	VMOVDQU64 (BP)(R15*1), Z21
	MOVQ      48(R14), BP
	VMOVDQU64 (BP)(R15*1), Z22
	MOVQ      72(R14), BP
	VMOVDQU64 (BP)(R15*1), Z23
	MOVQ      96(R14), BP
	VMOVDQU64 (BP)(R15*1), Z24
	MOVQ      120(R14), BP
	VMOVDQU64 (BP)(R15*1), Z25
	MOVQ      144(R14), BP
	VMOVDQU64 (BP)(R15*1), Z26
	MOVQ      168(R14), BP
	VMOVDQU64 (BP)(R15*1), Z27
	MOVQ      192(R14), BP
	VMOVDQU64 (BP)(R15*1), Z28
	MOVQ      216(R14), BP
	VMOVDQU64 (BP)(R15*1), Z29

	// Load and process 64 bytes from input 0 to 10 outputs
	VMOVDQU64      (BX), Z30
	ADDQ           $0x40, BX
	VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 1 to 10 outputs
	VMOVDQU64      (SI), Z30
	ADDQ           $0x40, SI
	VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
	VXORPD         Z20, Z31, Z20
	VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
	VXORPD         Z21, Z31, Z21
	VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
	VXORPD         Z22, Z31, Z22
	VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
	VXORPD         Z23, Z31, Z23
	VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
	VXORPD         Z24, Z31, Z24
	VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
	VXORPD         Z25, Z31, Z25
	VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
	VXORPD         Z26, Z31, Z26
	VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
	VXORPD         Z27, Z31, Z27
	VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
	VXORPD         Z28, Z31, Z28
	VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
	VXORPD         Z29, Z31, Z29

	// Load and process 64 bytes from input 2 to 10 outputs
	VMOVDQU64           (DI), Z30
	ADDQ                $0x40, DI
	VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 3 to 10 outputs
	VMOVDQU64           (R8), Z30
	ADDQ                $0x40, R8
	VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 4 to 10 outputs
	VMOVDQU64           (R9), Z30
	ADDQ                $0x40, R9
	VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 5 to 10 outputs
	VMOVDQU64           (R10), Z30
	ADDQ                $0x40, R10
	VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 6 to 10 outputs
	VMOVDQU64           (R11), Z30
	ADDQ                $0x40, R11
	VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 7 to 10 outputs
	VMOVDQU64           (R12), Z30
	ADDQ                $0x40, R12
	VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 8 to 10 outputs
	VMOVDQU64           (R13), Z30
	ADDQ                $0x40, R13
	VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Load and process 64 bytes from input 9 to 10 outputs
	VMOVDQU64           (DX), Z30
	ADDQ                $0x40, DX
	VGF2P8AFFINEQB.BCST $0x00, 720(CX), Z30, Z31
	VXORPD              Z20, Z31, Z20
	VGF2P8AFFINEQB.BCST $0x00, 728(CX), Z30, Z31
	VXORPD              Z21, Z31, Z21
	VGF2P8AFFINEQB.BCST $0x00, 736(CX), Z30, Z31
	VXORPD              Z22, Z31, Z22
	VGF2P8AFFINEQB.BCST $0x00, 744(CX), Z30, Z31
	VXORPD              Z23, Z31, Z23
	VGF2P8AFFINEQB.BCST $0x00, 752(CX), Z30, Z31
	VXORPD              Z24, Z31, Z24
	VGF2P8AFFINEQB.BCST $0x00, 760(CX), Z30, Z31
	VXORPD              Z25, Z31, Z25
	VGF2P8AFFINEQB.BCST $0x00, 768(CX), Z30, Z31
	VXORPD              Z26, Z31, Z26
	VGF2P8AFFINEQB.BCST $0x00, 776(CX), Z30, Z31
	VXORPD              Z27, Z31, Z27
	VGF2P8AFFINEQB.BCST $0x00, 784(CX), Z30, Z31
	VXORPD              Z28, Z31, Z28
	VGF2P8AFFINEQB.BCST $0x00, 792(CX), Z30, Z31
	VXORPD              Z29, Z31, Z29

	// Store 10 outputs
	MOVQ      (R14), BP
	VMOVDQU64 Z20, (BP)(R15*1)
	MOVQ      24(R14), BP
	VMOVDQU64 Z21, (BP)(R15*1)
	MOVQ      48(R14), BP
	VMOVDQU64 Z22, (BP)(R15*1)
	MOVQ      72(R14), BP
	VMOVDQU64 Z23, (BP)(R15*1)
	MOVQ      96(R14), BP
	VMOVDQU64 Z24, (BP)(R15*1)
	MOVQ      120(R14), BP
	VMOVDQU64 Z25, (BP)(R15*1)
	MOVQ      144(R14), BP
	VMOVDQU64 Z26, (BP)(R15*1)
	MOVQ      168(R14), BP
	VMOVDQU64 Z27, (BP)(R15*1)
	MOVQ      192(R14), BP
	VMOVDQU64 Z28, (BP)(R15*1)
	MOVQ      216(R14), BP
	VMOVDQU64 Z29, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x40, R15
	DECQ AX
	JNZ  mulGFNI_10x10_64Xor_loop
	VZEROUPPER

mulGFNI_10x10_64Xor_end:
	RET

// func mulAvxGFNI_10x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, GFNI
TEXT ·mulAvxGFNI_10x10Xor(SB), $8-88
	// Loading 4 of 100 tables to registers
	// Destination kept on stack
	// Full registers estimated 112 YMM used
	MOVQ         n+80(FP), AX
	MOVQ         matrix_base+0(FP), CX
	SHRQ         $0x05, AX
	TESTQ        AX, AX
	JZ           mulAvxGFNI_10x10Xor_end
	VBROADCASTSD (CX), Y0
	VBROADCASTSD 8(CX), Y1
	VBROADCASTSD 16(CX), Y2
	VBROADCASTSD 24(CX), Y3
	MOVQ         in_base+24(FP), DX
	MOVQ         (DX), BX
	MOVQ         24(DX), SI
	MOVQ         48(DX), DI
	MOVQ         72(DX), R8
	MOVQ         96(DX), R9
	MOVQ         120(DX), R10
	MOVQ         144(DX), R11
	MOVQ         168(DX), R12
	MOVQ         192(DX), R13
	MOVQ         216(DX), DX
	MOVQ         out_base+48(FP), R14
	MOVQ         out_base+48(FP), R14
	MOVQ         start+72(FP), R15

	// Add start offset to input
	ADDQ R15, BX
	ADDQ R15, SI
	ADDQ R15, DI
	ADDQ R15, R8
	ADDQ R15, R9
	ADDQ R15, R10
	ADDQ R15, R11
	ADDQ R15, R12
	ADDQ R15, R13
	ADDQ R15, DX

mulAvxGFNI_10x10Xor_loop:
	// Load 10 outputs
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y4
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y5
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y6
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y7
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y8
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y9
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y10
	MOVQ    168(R14), BP
	VMOVDQU (BP)(R15*1), Y11
	MOVQ    192(R14), BP
	VMOVDQU (BP)(R15*1), Y12
	MOVQ    216(R14), BP
	VMOVDQU (BP)(R15*1), Y13

	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU        (BX), Y14
	ADDQ           $0x20, BX
	VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   32(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   40(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   48(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   56(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   64(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   72(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU        (SI), Y14
	ADDQ           $0x20, SI
	VBROADCASTSD   80(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   88(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   96(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   104(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   112(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   120(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   128(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   136(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   144(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   152(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU        (DI), Y14
	ADDQ           $0x20, DI
	VBROADCASTSD   160(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   168(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   176(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   184(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   192(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   200(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   208(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   216(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   224(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   232(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU        (R8), Y14
	ADDQ           $0x20, R8
	VBROADCASTSD   240(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   248(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   256(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   264(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   272(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   280(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   288(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   296(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   304(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   312(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU        (R9), Y14
	ADDQ           $0x20, R9
	VBROADCASTSD   320(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   328(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   336(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   344(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   352(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   360(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   368(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   376(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   384(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   392(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU        (R10), Y14
	ADDQ           $0x20, R10
	VBROADCASTSD   400(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   408(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   416(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   424(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   432(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   440(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   448(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   456(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   464(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   472(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU        (R11), Y14
	ADDQ           $0x20, R11
	VBROADCASTSD   480(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   488(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   496(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   504(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   512(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   520(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   528(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   536(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   544(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   552(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU        (R12), Y14
	ADDQ           $0x20, R12
	VBROADCASTSD   560(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   568(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   576(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   584(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   592(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   600(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   608(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   616(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   624(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   632(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU        (R13), Y14
	ADDQ           $0x20, R13
	VBROADCASTSD   640(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   648(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   656(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   664(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   672(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   680(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   688(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   696(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   704(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   712(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Load and process 32 bytes from input 9 to 10 outputs
	VMOVDQU        (DX), Y14
	ADDQ           $0x20, DX
	VBROADCASTSD   720(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y4, Y15, Y4
	VBROADCASTSD   728(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y5, Y15, Y5
	VBROADCASTSD   736(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y6, Y15, Y6
	VBROADCASTSD   744(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y7, Y15, Y7
	VBROADCASTSD   752(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y8, Y15, Y8
	VBROADCASTSD   760(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y9, Y15, Y9
	VBROADCASTSD   768(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y10, Y15, Y10
	VBROADCASTSD   776(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y11, Y15, Y11
	VBROADCASTSD   784(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y12, Y15, Y12
	VBROADCASTSD   792(CX), Y15
	VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
	VXORPD         Y13, Y15, Y13

	// Store 10 outputs
	MOVQ    (R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y9, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y10, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y11, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y12, (BP)(R15*1)
	MOVQ    216(R14), BP
	VMOVDQU Y13, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxGFNI_10x10Xor_loop
	VZEROUPPER

mulAvxGFNI_10x10Xor_end:
	RET

// func mulAvxTwo_10x10Xor(matrix []byte, in [][]byte, out [][]byte, start int, n int)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·mulAvxTwo_10x10Xor(SB), NOSPLIT, $8-88
	// Loading no tables to registers
	// Destination kept on stack
	// Full registers estimated 215 YMM used
	MOVQ  n+80(FP), AX
	MOVQ  matrix_base+0(FP), CX
	SHRQ  $0x05, AX
	TESTQ AX, AX
	JZ    mulAvxTwo_10x10Xor_end
	MOVQ  in_base+24(FP), DX
	MOVQ  (DX), BX
	MOVQ  24(DX), SI
	MOVQ  48(DX), DI
	MOVQ  72(DX), R8
	MOVQ  96(DX), R9
	MOVQ  120(DX), R10
	MOVQ  144(DX), R11
	MOVQ  168(DX), R12
	MOVQ  192(DX), R13
	MOVQ  216(DX), DX
	MOVQ  out_base+48(FP), R14
	MOVQ  start+72(FP), R15

	// Add start offset to input
	ADDQ         R15, BX
	ADDQ         R15, SI
	ADDQ         R15, DI
	ADDQ         R15, R8
	ADDQ         R15, R9
	ADDQ         R15, R10
	ADDQ         R15, R11
	ADDQ         R15, R12
	ADDQ         R15, R13
	ADDQ         R15, DX
	MOVQ         $0x0000000f, BP
	MOVQ         BP, X10
	VPBROADCASTB X10, Y10

mulAvxTwo_10x10Xor_loop:
	// Load and process 32 bytes from input 0 to 10 outputs
	VMOVDQU (BX), Y13
	ADDQ    $0x20, BX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	MOVQ    (R14), BP
	VMOVDQU (BP)(R15*1), Y0
	VMOVDQU (CX), Y11
	VMOVDQU 32(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	MOVQ    24(R14), BP
	VMOVDQU (BP)(R15*1), Y1
	VMOVDQU 64(CX), Y11
	VMOVDQU 96(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	MOVQ    48(R14), BP
	VMOVDQU (BP)(R15*1), Y2
	VMOVDQU 128(CX), Y11
	VMOVDQU 160(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	MOVQ    72(R14), BP
	VMOVDQU (BP)(R15*1), Y3
	VMOVDQU 192(CX), Y11
	VMOVDQU 224(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	MOVQ    96(R14), BP
	VMOVDQU (BP)(R15*1), Y4
	VMOVDQU 256(CX), Y11
	VMOVDQU 288(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	MOVQ    120(R14), BP
	VMOVDQU (BP)(R15*1), Y5
	VMOVDQU 320(CX), Y11
	VMOVDQU 352(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	MOVQ    144(R14), BP
	VMOVDQU (BP)(R15*1), Y6
	VMOVDQU 384(CX), Y11
	VMOVDQU 416(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	MOVQ    168(R14), BP
	VMOVDQU (BP)(R15*1), Y7
	VMOVDQU 448(CX), Y11
	VMOVDQU 480(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	MOVQ    192(R14), BP
	VMOVDQU (BP)(R15*1), Y8
	VMOVDQU 512(CX), Y11
	VMOVDQU 544(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	MOVQ    216(R14), BP
	VMOVDQU (BP)(R15*1), Y9
	VMOVDQU 576(CX), Y11
	VMOVDQU 608(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 1 to 10 outputs
	VMOVDQU (SI), Y13
	ADDQ    $0x20, SI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 640(CX), Y11
	VMOVDQU 672(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 704(CX), Y11
	VMOVDQU 736(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 768(CX), Y11
	VMOVDQU 800(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 832(CX), Y11
	VMOVDQU 864(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 896(CX), Y11
	VMOVDQU 928(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 960(CX), Y11
	VMOVDQU 992(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1024(CX), Y11
	VMOVDQU 1056(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1088(CX), Y11
	VMOVDQU 1120(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1152(CX), Y11
	VMOVDQU 1184(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1216(CX), Y11
	VMOVDQU 1248(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 2 to 10 outputs
	VMOVDQU (DI), Y13
	ADDQ    $0x20, DI
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1280(CX), Y11
	VMOVDQU 1312(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1344(CX), Y11
	VMOVDQU 1376(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 1408(CX), Y11
	VMOVDQU 1440(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 1472(CX), Y11
	VMOVDQU 1504(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 1536(CX), Y11
	VMOVDQU 1568(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 1600(CX), Y11
	VMOVDQU 1632(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 1664(CX), Y11
	VMOVDQU 1696(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 1728(CX), Y11
	VMOVDQU 1760(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 1792(CX), Y11
	VMOVDQU 1824(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 1856(CX), Y11
	VMOVDQU 1888(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 3 to 10 outputs
	VMOVDQU (R8), Y13
	ADDQ    $0x20, R8
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 1920(CX), Y11
	VMOVDQU 1952(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 1984(CX), Y11
	VMOVDQU 2016(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2048(CX), Y11
	VMOVDQU 2080(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2112(CX), Y11
	VMOVDQU 2144(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2176(CX), Y11
	VMOVDQU 2208(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2240(CX), Y11
	VMOVDQU 2272(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2304(CX), Y11
	VMOVDQU 2336(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 2368(CX), Y11
	VMOVDQU 2400(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 2432(CX), Y11
	VMOVDQU 2464(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 2496(CX), Y11
	VMOVDQU 2528(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 4 to 10 outputs
	VMOVDQU (R9), Y13
	ADDQ    $0x20, R9
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 2560(CX), Y11
	VMOVDQU 2592(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 2624(CX), Y11
	VMOVDQU 2656(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 2688(CX), Y11
	VMOVDQU 2720(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 2752(CX), Y11
	VMOVDQU 2784(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 2816(CX), Y11
	VMOVDQU 2848(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 2880(CX), Y11
	VMOVDQU 2912(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 2944(CX), Y11
	VMOVDQU 2976(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3008(CX), Y11
	VMOVDQU 3040(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3072(CX), Y11
	VMOVDQU 3104(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3136(CX), Y11
	VMOVDQU 3168(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 5 to 10 outputs
	VMOVDQU (R10), Y13
	ADDQ    $0x20, R10
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3200(CX), Y11
	VMOVDQU 3232(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3264(CX), Y11
	VMOVDQU 3296(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3328(CX), Y11
	VMOVDQU 3360(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 3392(CX), Y11
	VMOVDQU 3424(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 3456(CX), Y11
	VMOVDQU 3488(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 3520(CX), Y11
	VMOVDQU 3552(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 3584(CX), Y11
	VMOVDQU 3616(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 3648(CX), Y11
	VMOVDQU 3680(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 3712(CX), Y11
	VMOVDQU 3744(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 3776(CX), Y11
	VMOVDQU 3808(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 6 to 10 outputs
	VMOVDQU (R11), Y13
	ADDQ    $0x20, R11
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 3840(CX), Y11
	VMOVDQU 3872(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 3904(CX), Y11
	VMOVDQU 3936(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 3968(CX), Y11
	VMOVDQU 4000(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4032(CX), Y11
	VMOVDQU 4064(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4096(CX), Y11
	VMOVDQU 4128(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4160(CX), Y11
	VMOVDQU 4192(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4224(CX), Y11
	VMOVDQU 4256(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4288(CX), Y11
	VMOVDQU 4320(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4352(CX), Y11
	VMOVDQU 4384(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 4416(CX), Y11
	VMOVDQU 4448(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 7 to 10 outputs
	VMOVDQU (R12), Y13
	ADDQ    $0x20, R12
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 4480(CX), Y11
	VMOVDQU 4512(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 4544(CX), Y11
	VMOVDQU 4576(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 4608(CX), Y11
	VMOVDQU 4640(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 4672(CX), Y11
	VMOVDQU 4704(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 4736(CX), Y11
	VMOVDQU 4768(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 4800(CX), Y11
	VMOVDQU 4832(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 4864(CX), Y11
	VMOVDQU 4896(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 4928(CX), Y11
	VMOVDQU 4960(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 4992(CX), Y11
	VMOVDQU 5024(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5056(CX), Y11
	VMOVDQU 5088(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 8 to 10 outputs
	VMOVDQU (R13), Y13
	ADDQ    $0x20, R13
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 5120(CX), Y11
	VMOVDQU 5152(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 5184(CX), Y11
	VMOVDQU 5216(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 5248(CX), Y11
	VMOVDQU 5280(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 5312(CX), Y11
	VMOVDQU 5344(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 5376(CX), Y11
	VMOVDQU 5408(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 5440(CX), Y11
	VMOVDQU 5472(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 5504(CX), Y11
	VMOVDQU 5536(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 5568(CX), Y11
	VMOVDQU 5600(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 5632(CX), Y11
	VMOVDQU 5664(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 5696(CX), Y11
	VMOVDQU 5728(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Load and process 32 bytes from input 9 to 10 outputs
	VMOVDQU (DX), Y13
	ADDQ    $0x20, DX
	VPSRLQ  $0x04, Y13, Y14
	VPAND   Y10, Y13, Y13
	VPAND   Y10, Y14, Y14
	VMOVDQU 5760(CX), Y11
	VMOVDQU 5792(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y0)
	VMOVDQU 5824(CX), Y11
	VMOVDQU 5856(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y1)
	VMOVDQU 5888(CX), Y11
	VMOVDQU 5920(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y2)
	VMOVDQU 5952(CX), Y11
	VMOVDQU 5984(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)
	VMOVDQU 6016(CX), Y11
	VMOVDQU 6048(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VMOVDQU 6080(CX), Y11
	VMOVDQU 6112(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)
	VMOVDQU 6144(CX), Y11
	VMOVDQU 6176(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU 6208(CX), Y11
	VMOVDQU 6240(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)
	VMOVDQU 6272(CX), Y11
	VMOVDQU 6304(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU 6336(CX), Y11
	VMOVDQU 6368(CX), Y12
	VPSHUFB Y13, Y11, Y11
	VPSHUFB Y14, Y12, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)

	// Store 10 outputs
	MOVQ    (R14), BP
	VMOVDQU Y0, (BP)(R15*1)
	MOVQ    24(R14), BP
	VMOVDQU Y1, (BP)(R15*1)
	MOVQ    48(R14), BP
	VMOVDQU Y2, (BP)(R15*1)
	MOVQ    72(R14), BP
	VMOVDQU Y3, (BP)(R15*1)
	MOVQ    96(R14), BP
	VMOVDQU Y4, (BP)(R15*1)
	MOVQ    120(R14), BP
	VMOVDQU Y5, (BP)(R15*1)
	MOVQ    144(R14), BP
	VMOVDQU Y6, (BP)(R15*1)
	MOVQ    168(R14), BP
	VMOVDQU Y7, (BP)(R15*1)
	MOVQ    192(R14), BP
	VMOVDQU Y8, (BP)(R15*1)
	MOVQ    216(R14), BP
	VMOVDQU Y9, (BP)(R15*1)

	// Prepare for next loop
	ADDQ $0x20, R15
	DECQ AX
	JNZ  mulAvxTwo_10x10Xor_loop
	VZEROUPPER

mulAvxTwo_10x10Xor_end:
	RET

// func ifftDIT2_avx2(x []byte, y []byte, table *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT2_avx2(SB), NOSPLIT, $0-56
	MOVQ           table+48(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 64(AX), Y1
	VBROADCASTI128 16(AX), Y2
	VBROADCASTI128 80(AX), Y3
	VBROADCASTI128 32(AX), Y4
	VBROADCASTI128 96(AX), Y5
	VBROADCASTI128 48(AX), Y6
	VBROADCASTI128 112(AX), Y7
	MOVQ           x_len+8(FP), AX
	MOVQ           x_base+0(FP), CX
	MOVQ           y_base+24(FP), DX
	MOVQ           $0x0000000f, BX
	MOVQ           BX, X8
	VPBROADCASTB   X8, Y8

loop:
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y12
	VPXOR   Y11, Y9, Y11
	VPXOR   Y12, Y10, Y12
	VMOVDQU Y11, (DX)
	VMOVDQU Y12, 32(DX)
	VPSRLQ  $0x04, Y11, Y13
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y13, Y13
	VPSHUFB Y11, Y0, Y14
	VPSHUFB Y11, Y1, Y11
	VPSHUFB Y13, Y2, Y15
	VPSHUFB Y13, Y3, Y13
	VPXOR   Y14, Y15, Y14
	VPXOR   Y11, Y13, Y11
	VPAND   Y12, Y8, Y13
	VPSRLQ  $0x04, Y12, Y12
	VPAND   Y8, Y12, Y12
	VPSHUFB Y13, Y4, Y15
	VPSHUFB Y13, Y5, Y13
	VPXOR   Y14, Y15, Y14
	VPXOR   Y11, Y13, Y11
	VPSHUFB Y12, Y6, Y15
	VPSHUFB Y12, Y7, Y13
	XOR3WAY( $0x00, Y14, Y15, Y9)
	XOR3WAY( $0x00, Y11, Y13, Y10)
	VMOVDQU Y9, (CX)
	VMOVDQU Y10, 32(CX)
	ADDQ    $0x40, CX
	ADDQ    $0x40, DX
	SUBQ    $0x40, AX
	JNZ     loop
	VZEROUPPER
	RET

// func fftDIT2_avx2(x []byte, y []byte, table *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT2_avx2(SB), NOSPLIT, $0-56
	MOVQ           table+48(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 64(AX), Y1
	VBROADCASTI128 16(AX), Y2
	VBROADCASTI128 80(AX), Y3
	VBROADCASTI128 32(AX), Y4
	VBROADCASTI128 96(AX), Y5
	VBROADCASTI128 48(AX), Y6
	VBROADCASTI128 112(AX), Y7
	MOVQ           x_len+8(FP), AX
	MOVQ           x_base+0(FP), CX
	MOVQ           y_base+24(FP), DX
	MOVQ           $0x0000000f, BX
	MOVQ           BX, X8
	VPBROADCASTB   X8, Y8

loop:
	VMOVDQU (CX), Y9
	VMOVDQU 32(CX), Y10
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y12
	VPSRLQ  $0x04, Y11, Y13
	VPAND   Y8, Y11, Y11
	VPAND   Y8, Y13, Y13
	VPSHUFB Y11, Y0, Y14
	VPSHUFB Y11, Y1, Y11
	VPSHUFB Y13, Y2, Y15
	VPSHUFB Y13, Y3, Y13
	VPXOR   Y14, Y15, Y14
	VPXOR   Y11, Y13, Y11
	VPAND   Y12, Y8, Y13
	VPSRLQ  $0x04, Y12, Y12
	VPAND   Y8, Y12, Y12
	VPSHUFB Y13, Y4, Y15
	VPSHUFB Y13, Y5, Y13
	VPXOR   Y14, Y15, Y14
	VPXOR   Y11, Y13, Y11
	VPSHUFB Y12, Y6, Y15
	VPSHUFB Y12, Y7, Y13
	XOR3WAY( $0x00, Y14, Y15, Y9)
	XOR3WAY( $0x00, Y11, Y13, Y10)
	VMOVDQU Y9, (CX)
	VMOVDQU Y10, 32(CX)
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y12
	VPXOR   Y11, Y9, Y11
	VPXOR   Y12, Y10, Y12
	VMOVDQU Y11, (DX)
	VMOVDQU Y12, 32(DX)
	ADDQ    $0x40, CX
	ADDQ    $0x40, DX
	SUBQ    $0x40, AX
	JNZ     loop
	VZEROUPPER
	RET

// func mulgf16_avx2(x []byte, y []byte, table *[128]uint8)
// Requires: AVX, AVX2, SSE2
TEXT ·mulgf16_avx2(SB), NOSPLIT, $0-56
	MOVQ           table+48(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 64(AX), Y1
	VBROADCASTI128 16(AX), Y2
	VBROADCASTI128 80(AX), Y3
	VBROADCASTI128 32(AX), Y4
	VBROADCASTI128 96(AX), Y5
	VBROADCASTI128 48(AX), Y6
	VBROADCASTI128 112(AX), Y7
	MOVQ           x_len+8(FP), AX
	MOVQ           x_base+0(FP), CX
	MOVQ           y_base+24(FP), DX
	MOVQ           $0x0000000f, BX
	MOVQ           BX, X8
	VPBROADCASTB   X8, Y8

loop:
	VMOVDQU (DX), Y9
	VMOVDQU 32(DX), Y10
	VPSRLQ  $0x04, Y9, Y11
	VPAND   Y8, Y9, Y9
	VPAND   Y8, Y11, Y11
	VPSHUFB Y9, Y0, Y12
	VPSHUFB Y9, Y1, Y9
	VPSHUFB Y11, Y2, Y13
	VPSHUFB Y11, Y3, Y11
	VPXOR   Y12, Y13, Y12
	VPXOR   Y9, Y11, Y9
	VPAND   Y10, Y8, Y11
	VPSRLQ  $0x04, Y10, Y10
	VPAND   Y8, Y10, Y10
	VPSHUFB Y11, Y4, Y13
	VPSHUFB Y11, Y5, Y11
	VPXOR   Y12, Y13, Y12
	VPXOR   Y9, Y11, Y9
	VPSHUFB Y10, Y6, Y13
	VPSHUFB Y10, Y7, Y11
	VPXOR   Y12, Y13, Y12
	VPXOR   Y9, Y11, Y9
	VMOVDQU Y12, (CX)
	VMOVDQU Y9, 32(CX)
	ADDQ    $0x40, CX
	ADDQ    $0x40, DX
	SUBQ    $0x40, AX
	JNZ     loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_0(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx512_0(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), DX
	VBROADCASTI128 (DX), Y1
	VBROADCASTI128 64(DX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(DX), Y1
	VBROADCASTI128 80(DX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(DX), Y1
	VBROADCASTI128 96(DX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(DX), Y1
	VBROADCASTI128 112(DX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), DX
	MOVQ           8(DX), BX
	XORQ           SI, SI
	MOVQ           (DX)(SI*1), DI
	ADDQ           AX, SI
	MOVQ           (DX)(SI*1), R8
	ADDQ           AX, SI
	MOVQ           (DX)(SI*1), R9
	ADDQ           AX, SI
	MOVQ           (DX)(SI*1), AX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VPSRLQ         $0x04, Y3, Y6
	VPAND          Y0, Y3, Y5
	VPAND          Y0, Y6, Y6
	VPSHUFB        Y5, Y24, Y7
	VPSHUFB        Y5, Y25, Y5
	VPSHUFB        Y6, Y26, Y8
	VPSHUFB        Y6, Y27, Y6
	VPXOR          Y7, Y8, Y7
	VPXOR          Y5, Y6, Y5
	VPAND          Y4, Y0, Y6
	VPSRLQ         $0x04, Y4, Y8
	VPAND          Y0, Y8, Y8
	VPSHUFB        Y6, Y28, Y9
	VPSHUFB        Y6, Y29, Y6
	VPXOR          Y7, Y9, Y7
	VPXOR          Y5, Y6, Y5
	VPSHUFB        Y8, Y30, Y9
	VPSHUFB        Y8, Y31, Y6
	VPTERNLOGD     $0x96, Y7, Y9, Y1
	VPTERNLOGD     $0x96, Y5, Y6, Y2
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (AX), Y7
	VMOVDQU        32(AX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	VPTERNLOGD     $0x96, Y11, Y13, Y5
	VPTERNLOGD     $0x96, Y9, Y10, Y6
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VPSHUFB        Y9, Y16, Y11
	VPSHUFB        Y9, Y17, Y9
	VPSHUFB        Y10, Y18, Y12
	VPSHUFB        Y10, Y19, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VPSHUFB        Y10, Y20, Y13
	VPSHUFB        Y10, Y21, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VPSHUFB        Y12, Y22, Y13
	VPSHUFB        Y12, Y23, Y10
	VPTERNLOGD     $0x96, Y11, Y13, Y1
	VPTERNLOGD     $0x96, Y9, Y10, Y2
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VPSHUFB        Y9, Y16, Y11
	VPSHUFB        Y9, Y17, Y9
	VPSHUFB        Y10, Y18, Y12
	VPSHUFB        Y10, Y19, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VPSHUFB        Y10, Y20, Y13
	VPSHUFB        Y10, Y21, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VPSHUFB        Y12, Y22, Y13
	VPSHUFB        Y12, Y23, Y10
	VPTERNLOGD     $0x96, Y11, Y13, Y3
	VPTERNLOGD     $0x96, Y9, Y10, Y4
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (AX)
	VMOVDQU        Y8, 32(AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_0(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx512_0(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), DX
	VBROADCASTI128 (DX), Y1
	VBROADCASTI128 64(DX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(DX), Y1
	VBROADCASTI128 80(DX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(DX), Y1
	VBROADCASTI128 96(DX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(DX), Y1
	VBROADCASTI128 112(DX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), DX
	MOVQ           8(DX), BX
	XORQ           SI, SI
	MOVQ           (DX)(SI*1), DI
	ADDQ           AX, SI
	MOVQ           (DX)(SI*1), R8
	ADDQ           AX, SI
	MOVQ           (DX)(SI*1), R9
	ADDQ           AX, SI
	MOVQ           (DX)(SI*1), AX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VMOVDQU        (AX), Y7
	VMOVDQU        32(AX), Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VPSHUFB        Y9, Y16, Y11
	VPSHUFB        Y9, Y17, Y9
	VPSHUFB        Y10, Y18, Y12
	VPSHUFB        Y10, Y19, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VPSHUFB        Y10, Y20, Y13
	VPSHUFB        Y10, Y21, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VPSHUFB        Y12, Y22, Y13
	VPSHUFB        Y12, Y23, Y10
	VPTERNLOGD     $0x96, Y11, Y13, Y1
	VPTERNLOGD     $0x96, Y9, Y10, Y2
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VPSHUFB        Y9, Y16, Y11
	VPSHUFB        Y9, Y17, Y9
	VPSHUFB        Y10, Y18, Y12
	VPSHUFB        Y10, Y19, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VPSHUFB        Y10, Y20, Y13
	VPSHUFB        Y10, Y21, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VPSHUFB        Y12, Y22, Y13
	VPSHUFB        Y12, Y23, Y10
	VPTERNLOGD     $0x96, Y11, Y13, Y3
	VPTERNLOGD     $0x96, Y9, Y10, Y4
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y3, Y10
	VPAND          Y0, Y3, Y9
	VPAND          Y0, Y10, Y10
	VPSHUFB        Y9, Y24, Y11
	VPSHUFB        Y9, Y25, Y9
	VPSHUFB        Y10, Y26, Y12
	VPSHUFB        Y10, Y27, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y4, Y0, Y10
	VPSRLQ         $0x04, Y4, Y12
	VPAND          Y0, Y12, Y12
	VPSHUFB        Y10, Y28, Y13
	VPSHUFB        Y10, Y29, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VPSHUFB        Y12, Y30, Y13
	VPSHUFB        Y12, Y31, Y10
	VPTERNLOGD     $0x96, Y11, Y13, Y1
	VPTERNLOGD     $0x96, Y9, Y10, Y2
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VPSRLQ         $0x04, Y7, Y2
	VPAND          Y0, Y7, Y1
	VPAND          Y0, Y2, Y2
	VBROADCASTI128 (CX), Y3
	VBROADCASTI128 64(CX), Y4
	VPSHUFB        Y1, Y3, Y3
	VPSHUFB        Y1, Y4, Y1
	VBROADCASTI128 16(CX), Y4
	VBROADCASTI128 80(CX), Y9
	VPSHUFB        Y2, Y4, Y4
	VPSHUFB        Y2, Y9, Y2
	VPXOR          Y3, Y4, Y3
	VPXOR          Y1, Y2, Y1
	VPAND          Y8, Y0, Y2
	VPSRLQ         $0x04, Y8, Y4
	VPAND          Y0, Y4, Y4
	VBROADCASTI128 32(CX), Y9
	VBROADCASTI128 96(CX), Y10
	VPSHUFB        Y2, Y9, Y9
	VPSHUFB        Y2, Y10, Y2
	VPXOR          Y3, Y9, Y3
	VPXOR          Y1, Y2, Y1
	VBROADCASTI128 48(CX), Y9
	VBROADCASTI128 112(CX), Y2
	VPSHUFB        Y4, Y9, Y9
	VPSHUFB        Y4, Y2, Y2
	VPTERNLOGD     $0x96, Y3, Y9, Y5
	VPTERNLOGD     $0x96, Y1, Y2, Y6
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (AX)
	VMOVDQU        Y8, 32(AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_1(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx512_1(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), AX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (CX), Y1
	VBROADCASTI128 64(CX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(CX), Y1
	VBROADCASTI128 80(CX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(CX), Y1
	VBROADCASTI128 96(CX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(CX), Y1
	VBROADCASTI128 112(CX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y24, Y11
	VPSHUFB    Y9, Y25, Y9
	VPSHUFB    Y10, Y26, Y12
	VPSHUFB    Y10, Y27, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y28, Y13
	VPSHUFB    Y10, Y29, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y30, Y13
	VPSHUFB    Y12, Y31, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y5
	VPTERNLOGD $0x96, Y9, Y10, Y6
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPSRLQ     $0x04, Y5, Y10
	VPAND      Y0, Y5, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y6, Y0, Y10
	VPSRLQ     $0x04, Y6, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y3
	VPTERNLOGD $0x96, Y9, Y10, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_1(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx512_1(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), DX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (CX), Y1
	VBROADCASTI128 64(CX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(CX), Y1
	VBROADCASTI128 80(CX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(CX), Y1
	VBROADCASTI128 96(CX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(CX), Y1
	VBROADCASTI128 112(CX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPSRLQ     $0x04, Y3, Y10
	VPAND      Y0, Y3, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y4, Y0, Y10
	VPSRLQ     $0x04, Y4, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VPSRLQ     $0x04, Y7, Y2
	VPAND      Y0, Y7, Y1
	VPAND      Y0, Y2, Y2
	VPSHUFB    Y1, Y24, Y3
	VPSHUFB    Y1, Y25, Y1
	VPSHUFB    Y2, Y26, Y4
	VPSHUFB    Y2, Y27, Y2
	VPXOR      Y3, Y4, Y3
	VPXOR      Y1, Y2, Y1
	VPAND      Y8, Y0, Y2
	VPSRLQ     $0x04, Y8, Y4
	VPAND      Y0, Y4, Y4
	VPSHUFB    Y2, Y28, Y9
	VPSHUFB    Y2, Y29, Y2
	VPXOR      Y3, Y9, Y3
	VPXOR      Y1, Y2, Y1
	VPSHUFB    Y4, Y30, Y9
	VPSHUFB    Y4, Y31, Y2
	VPTERNLOGD $0x96, Y3, Y9, Y5
	VPTERNLOGD $0x96, Y1, Y2, Y6
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_2(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx512_2(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (CX), Y1
	VBROADCASTI128 64(CX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(CX), Y1
	VBROADCASTI128 80(CX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(CX), Y1
	VBROADCASTI128 96(CX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(CX), Y1
	VBROADCASTI128 112(CX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VPSRLQ     $0x04, Y3, Y6
	VPAND      Y0, Y3, Y5
	VPAND      Y0, Y6, Y6
	VPSHUFB    Y5, Y24, Y7
	VPSHUFB    Y5, Y25, Y5
	VPSHUFB    Y6, Y26, Y8
	VPSHUFB    Y6, Y27, Y6
	VPXOR      Y7, Y8, Y7
	VPXOR      Y5, Y6, Y5
	VPAND      Y4, Y0, Y6
	VPSRLQ     $0x04, Y4, Y8
	VPAND      Y0, Y8, Y8
	VPSHUFB    Y6, Y28, Y9
	VPSHUFB    Y6, Y29, Y6
	VPXOR      Y7, Y9, Y7
	VPXOR      Y5, Y6, Y5
	VPSHUFB    Y8, Y30, Y9
	VPSHUFB    Y8, Y31, Y6
	VPTERNLOGD $0x96, Y7, Y9, Y1
	VPTERNLOGD $0x96, Y5, Y6, Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPSRLQ     $0x04, Y5, Y10
	VPAND      Y0, Y5, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y6, Y0, Y10
	VPSRLQ     $0x04, Y6, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y3
	VPTERNLOGD $0x96, Y9, Y10, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_2(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx512_2(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), AX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (CX), Y1
	VBROADCASTI128 64(CX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(CX), Y1
	VBROADCASTI128 80(CX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(CX), Y1
	VBROADCASTI128 96(CX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(CX), Y1
	VBROADCASTI128 112(CX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPSRLQ     $0x04, Y5, Y10
	VPAND      Y0, Y5, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y6, Y0, Y10
	VPSRLQ     $0x04, Y6, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y3
	VPTERNLOGD $0x96, Y9, Y10, Y4
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VPSRLQ     $0x04, Y7, Y2
	VPAND      Y0, Y7, Y1
	VPAND      Y0, Y2, Y2
	VPSHUFB    Y1, Y24, Y3
	VPSHUFB    Y1, Y25, Y1
	VPSHUFB    Y2, Y26, Y4
	VPSHUFB    Y2, Y27, Y2
	VPXOR      Y3, Y4, Y3
	VPXOR      Y1, Y2, Y1
	VPAND      Y8, Y0, Y2
	VPSRLQ     $0x04, Y8, Y4
	VPAND      Y0, Y4, Y4
	VPSHUFB    Y2, Y28, Y9
	VPSHUFB    Y2, Y29, Y2
	VPXOR      Y3, Y9, Y3
	VPXOR      Y1, Y2, Y1
	VPSHUFB    Y4, Y30, Y9
	VPSHUFB    Y4, Y31, Y2
	VPTERNLOGD $0x96, Y3, Y9, Y5
	VPTERNLOGD $0x96, Y1, Y2, Y6
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_3(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx512_3(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), AX
	MOVQ           table02+48(FP), AX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPSRLQ     $0x04, Y5, Y10
	VPAND      Y0, Y5, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y6, Y0, Y10
	VPSRLQ     $0x04, Y6, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y3
	VPTERNLOGD $0x96, Y9, Y10, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_3(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx512_3(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), AX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VPSRLQ     $0x04, Y7, Y2
	VPAND      Y0, Y7, Y1
	VPAND      Y0, Y2, Y2
	VPSHUFB    Y1, Y16, Y3
	VPSHUFB    Y1, Y17, Y1
	VPSHUFB    Y2, Y18, Y4
	VPSHUFB    Y2, Y19, Y2
	VPXOR      Y3, Y4, Y3
	VPXOR      Y1, Y2, Y1
	VPAND      Y8, Y0, Y2
	VPSRLQ     $0x04, Y8, Y4
	VPAND      Y0, Y4, Y4
	VPSHUFB    Y2, Y20, Y9
	VPSHUFB    Y2, Y21, Y2
	VPXOR      Y3, Y9, Y3
	VPXOR      Y1, Y2, Y1
	VPSHUFB    Y4, Y22, Y9
	VPSHUFB    Y4, Y23, Y2
	VPTERNLOGD $0x96, Y3, Y9, Y5
	VPTERNLOGD $0x96, Y1, Y2, Y6
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_4(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx512_4(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), DX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (CX), Y1
	VBROADCASTI128 64(CX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(CX), Y1
	VBROADCASTI128 80(CX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(CX), Y1
	VBROADCASTI128 96(CX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(CX), Y1
	VBROADCASTI128 112(CX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VPSRLQ     $0x04, Y3, Y6
	VPAND      Y0, Y3, Y5
	VPAND      Y0, Y6, Y6
	VPSHUFB    Y5, Y16, Y7
	VPSHUFB    Y5, Y17, Y5
	VPSHUFB    Y6, Y18, Y8
	VPSHUFB    Y6, Y19, Y6
	VPXOR      Y7, Y8, Y7
	VPXOR      Y5, Y6, Y5
	VPAND      Y4, Y0, Y6
	VPSRLQ     $0x04, Y4, Y8
	VPAND      Y0, Y8, Y8
	VPSHUFB    Y6, Y20, Y9
	VPSHUFB    Y6, Y21, Y6
	VPXOR      Y7, Y9, Y7
	VPXOR      Y5, Y6, Y5
	VPSHUFB    Y8, Y22, Y9
	VPSHUFB    Y8, Y23, Y6
	VPTERNLOGD $0x96, Y7, Y9, Y1
	VPTERNLOGD $0x96, Y5, Y6, Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y24, Y11
	VPSHUFB    Y9, Y25, Y9
	VPSHUFB    Y10, Y26, Y12
	VPSHUFB    Y10, Y27, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y28, Y13
	VPSHUFB    Y10, Y29, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y30, Y13
	VPSHUFB    Y12, Y31, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y5
	VPTERNLOGD $0x96, Y9, Y10, Y6
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_4(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx512_4(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (CX), Y1
	VBROADCASTI128 64(CX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(CX), Y1
	VBROADCASTI128 80(CX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(CX), Y1
	VBROADCASTI128 96(CX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(CX), Y1
	VBROADCASTI128 112(CX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z24
	VMOVAPS        Z0, Z25
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z26
	VMOVAPS        Z0, Z27
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z28
	VMOVAPS        Z0, Z29
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z30
	VMOVAPS        Z0, Z31
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPSRLQ     $0x04, Y5, Y10
	VPAND      Y0, Y5, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y6, Y0, Y10
	VPSRLQ     $0x04, Y6, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y3
	VPTERNLOGD $0x96, Y9, Y10, Y4
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPSRLQ     $0x04, Y3, Y10
	VPAND      Y0, Y3, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y24, Y11
	VPSHUFB    Y9, Y25, Y9
	VPSHUFB    Y10, Y26, Y12
	VPSHUFB    Y10, Y27, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y4, Y0, Y10
	VPSRLQ     $0x04, Y4, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y28, Y13
	VPSHUFB    Y10, Y29, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y30, Y13
	VPSHUFB    Y12, Y31, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_5(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx512_5(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), AX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y5
	VPTERNLOGD $0x96, Y9, Y10, Y6
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_5(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx512_5(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPSRLQ     $0x04, Y3, Y10
	VPAND      Y0, Y3, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y4, Y0, Y10
	VPSRLQ     $0x04, Y4, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_6(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx512_6(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), CX
	MOVQ           table02+48(FP), CX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VPSRLQ     $0x04, Y3, Y6
	VPAND      Y0, Y3, Y5
	VPAND      Y0, Y6, Y6
	VPSHUFB    Y5, Y16, Y7
	VPSHUFB    Y5, Y17, Y5
	VPSHUFB    Y6, Y18, Y8
	VPSHUFB    Y6, Y19, Y6
	VPXOR      Y7, Y8, Y7
	VPXOR      Y5, Y6, Y5
	VPAND      Y4, Y0, Y6
	VPSRLQ     $0x04, Y4, Y8
	VPAND      Y0, Y8, Y8
	VPSHUFB    Y6, Y20, Y9
	VPSHUFB    Y6, Y21, Y6
	VPXOR      Y7, Y9, Y7
	VPXOR      Y5, Y6, Y5
	VPSHUFB    Y8, Y22, Y9
	VPSHUFB    Y8, Y23, Y6
	VPTERNLOGD $0x96, Y7, Y9, Y1
	VPTERNLOGD $0x96, Y5, Y6, Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_6(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx512_6(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ           table01+32(FP), AX
	MOVQ           table23+40(FP), AX
	MOVQ           table02+48(FP), AX
	VBROADCASTI128 (AX), Y1
	VBROADCASTI128 64(AX), Y0
	VMOVAPS        Z1, Z16
	VMOVAPS        Z0, Z17
	VBROADCASTI128 16(AX), Y1
	VBROADCASTI128 80(AX), Y0
	VMOVAPS        Z1, Z18
	VMOVAPS        Z0, Z19
	VBROADCASTI128 32(AX), Y1
	VBROADCASTI128 96(AX), Y0
	VMOVAPS        Z1, Z20
	VMOVAPS        Z0, Z21
	VBROADCASTI128 48(AX), Y1
	VBROADCASTI128 112(AX), Y0
	VMOVAPS        Z1, Z22
	VMOVAPS        Z0, Z23
	MOVQ           $0x0000000f, AX
	MOVQ           AX, X0
	VPBROADCASTB   X0, Y0
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX

loop:
	VMOVDQU    (SI), Y1
	VMOVDQU    32(SI), Y2
	VMOVDQU    (R8), Y5
	VMOVDQU    32(R8), Y6
	VMOVDQU    (DI), Y3
	VMOVDQU    32(DI), Y4
	VMOVDQU    (AX), Y7
	VMOVDQU    32(AX), Y8
	VPSRLQ     $0x04, Y5, Y10
	VPAND      Y0, Y5, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y6, Y0, Y10
	VPSRLQ     $0x04, Y6, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y1
	VPTERNLOGD $0x96, Y9, Y10, Y2
	VPSRLQ     $0x04, Y7, Y10
	VPAND      Y0, Y7, Y9
	VPAND      Y0, Y10, Y10
	VPSHUFB    Y9, Y16, Y11
	VPSHUFB    Y9, Y17, Y9
	VPSHUFB    Y10, Y18, Y12
	VPSHUFB    Y10, Y19, Y10
	VPXOR      Y11, Y12, Y11
	VPXOR      Y9, Y10, Y9
	VPAND      Y8, Y0, Y10
	VPSRLQ     $0x04, Y8, Y12
	VPAND      Y0, Y12, Y12
	VPSHUFB    Y10, Y20, Y13
	VPSHUFB    Y10, Y21, Y10
	VPXOR      Y11, Y13, Y11
	VPXOR      Y9, Y10, Y9
	VPSHUFB    Y12, Y22, Y13
	VPSHUFB    Y12, Y23, Y10
	VPTERNLOGD $0x96, Y11, Y13, Y3
	VPTERNLOGD $0x96, Y9, Y10, Y4
	VPXOR      Y1, Y5, Y5
	VPXOR      Y2, Y6, Y6
	VPXOR      Y3, Y7, Y7
	VPXOR      Y4, Y8, Y8
	VPXOR      Y1, Y3, Y3
	VPXOR      Y2, Y4, Y4
	VMOVDQU    Y1, (SI)
	VMOVDQU    Y2, 32(SI)
	ADDQ       $0x40, SI
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y4, 32(DI)
	ADDQ       $0x40, DI
	VPXOR      Y5, Y7, Y7
	VPXOR      Y6, Y8, Y8
	VMOVDQU    Y5, (R8)
	VMOVDQU    Y6, 32(R8)
	ADDQ       $0x40, R8
	VMOVDQU    Y7, (AX)
	VMOVDQU    Y8, 32(AX)
	ADDQ       $0x40, AX
	SUBQ       $0x40, DX
	JNZ        loop
	VZEROUPPER
	RET

// func ifftDIT4_avx512_7(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, SSE2
TEXT ·ifftDIT4_avx512_7(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), AX
	MOVQ         $0x0000000f, AX
	MOVQ         AX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), AX
	MOVQ         work_base+0(FP), CX
	MOVQ         8(CX), DX
	XORQ         BX, BX
	MOVQ         (CX)(BX*1), SI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), DI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), R8
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), AX

loop:
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y1
	VMOVDQU (DI), Y2
	VMOVDQU 32(DI), Y3
	VPXOR   Y0, Y2, Y2
	VPXOR   Y1, Y3, Y3
	VMOVDQU (R8), Y4
	VMOVDQU 32(R8), Y5
	VMOVDQU (AX), Y6
	VMOVDQU 32(AX), Y7
	VPXOR   Y4, Y6, Y6
	VPXOR   Y5, Y7, Y7
	VPXOR   Y0, Y4, Y4
	VPXOR   Y1, Y5, Y5
	VPXOR   Y2, Y6, Y6
	VPXOR   Y3, Y7, Y7
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y4, (R8)
	VMOVDQU Y5, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y6, (AX)
	VMOVDQU Y7, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JNZ     loop
	VZEROUPPER
	RET

// func fftDIT4_avx512_7(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, SSE2
TEXT ·fftDIT4_avx512_7(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), AX
	MOVQ         $0x0000000f, AX
	MOVQ         AX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), AX
	MOVQ         work_base+0(FP), CX
	MOVQ         8(CX), DX
	XORQ         BX, BX
	MOVQ         (CX)(BX*1), SI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), DI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), R8
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), AX

loop:
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y1
	VMOVDQU (R8), Y4
	VMOVDQU 32(R8), Y5
	VMOVDQU (DI), Y2
	VMOVDQU 32(DI), Y3
	VMOVDQU (AX), Y6
	VMOVDQU 32(AX), Y7
	VPXOR   Y0, Y4, Y4
	VPXOR   Y1, Y5, Y5
	VPXOR   Y2, Y6, Y6
	VPXOR   Y3, Y7, Y7
	VPXOR   Y0, Y2, Y2
	VPXOR   Y1, Y3, Y3
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VPXOR   Y4, Y6, Y6
	VPXOR   Y5, Y7, Y7
	VMOVDQU Y4, (R8)
	VMOVDQU Y5, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y6, (AX)
	VMOVDQU Y7, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JNZ     loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_0(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx2_0(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), DX
	MOVQ         $0x0000000f, BX
	MOVQ         BX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), BX
	MOVQ         work_base+0(FP), SI
	MOVQ         8(SI), DI
	XORQ         R8, R8
	MOVQ         (SI)(R8*1), R9
	ADDQ         BX, R8
	MOVQ         (SI)(R8*1), R10
	ADDQ         BX, R8
	MOVQ         (SI)(R8*1), R11
	ADDQ         BX, R8
	MOVQ         (SI)(R8*1), BX

loop:
	VMOVDQU        (R9), Y1
	VMOVDQU        32(R9), Y2
	VMOVDQU        (R10), Y3
	VMOVDQU        32(R10), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VPSRLQ         $0x04, Y3, Y6
	VPAND          Y0, Y3, Y5
	VPAND          Y0, Y6, Y6
	VBROADCASTI128 (AX), Y7
	VBROADCASTI128 64(AX), Y8
	VPSHUFB        Y5, Y7, Y7
	VPSHUFB        Y5, Y8, Y5
	VBROADCASTI128 16(AX), Y8
	VBROADCASTI128 80(AX), Y9
	VPSHUFB        Y6, Y8, Y8
	VPSHUFB        Y6, Y9, Y6
	VPXOR          Y7, Y8, Y7
	VPXOR          Y5, Y6, Y5
	VPAND          Y4, Y0, Y6
	VPSRLQ         $0x04, Y4, Y8
	VPAND          Y0, Y8, Y8
	VBROADCASTI128 32(AX), Y9
	VBROADCASTI128 96(AX), Y10
	VPSHUFB        Y6, Y9, Y9
	VPSHUFB        Y6, Y10, Y6
	VPXOR          Y7, Y9, Y7
	VPXOR          Y5, Y6, Y5
	VBROADCASTI128 48(AX), Y9
	VBROADCASTI128 112(AX), Y6
	VPSHUFB        Y8, Y9, Y9
	VPSHUFB        Y8, Y6, Y6
	XOR3WAY(     $0x00, Y7, Y9, Y1)
	XOR3WAY(     $0x00, Y5, Y6, Y2)
	VMOVDQU        (R11), Y5
	VMOVDQU        32(R11), Y6
	VMOVDQU        (BX), Y7
	VMOVDQU        32(BX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y5)
	XOR3WAY(     $0x00, Y9, Y10, Y6)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (DX), Y11
	VBROADCASTI128 64(DX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(DX), Y12
	VBROADCASTI128 80(DX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(DX), Y13
	VBROADCASTI128 96(DX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(DX), Y13
	VBROADCASTI128 112(DX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (DX), Y11
	VBROADCASTI128 64(DX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(DX), Y12
	VBROADCASTI128 80(DX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(DX), Y13
	VBROADCASTI128 96(DX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(DX), Y13
	VBROADCASTI128 112(DX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VMOVDQU        Y1, (R9)
	VMOVDQU        Y2, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y3, (R10)
	VMOVDQU        Y4, 32(R10)
	ADDQ           $0x40, R10
	VMOVDQU        Y5, (R11)
	VMOVDQU        Y6, 32(R11)
	ADDQ           $0x40, R11
	VMOVDQU        Y7, (BX)
	VMOVDQU        Y8, 32(BX)
	ADDQ           $0x40, BX
	SUBQ           $0x40, DI
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_0(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx2_0(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), DX
	MOVQ         $0x0000000f, BX
	MOVQ         BX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), BX
	MOVQ         work_base+0(FP), SI
	MOVQ         8(SI), DI
	XORQ         R8, R8
	MOVQ         (SI)(R8*1), R9
	ADDQ         BX, R8
	MOVQ         (SI)(R8*1), R10
	ADDQ         BX, R8
	MOVQ         (SI)(R8*1), R11
	ADDQ         BX, R8
	MOVQ         (SI)(R8*1), BX

loop:
	VMOVDQU        (R9), Y1
	VMOVDQU        32(R9), Y2
	VMOVDQU        (R11), Y5
	VMOVDQU        32(R11), Y6
	VMOVDQU        (R10), Y3
	VMOVDQU        32(R10), Y4
	VMOVDQU        (BX), Y7
	VMOVDQU        32(BX), Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (DX), Y11
	VBROADCASTI128 64(DX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(DX), Y12
	VBROADCASTI128 80(DX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(DX), Y13
	VBROADCASTI128 96(DX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(DX), Y13
	VBROADCASTI128 112(DX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (DX), Y11
	VBROADCASTI128 64(DX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(DX), Y12
	VBROADCASTI128 80(DX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(DX), Y13
	VBROADCASTI128 96(DX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(DX), Y13
	VBROADCASTI128 112(DX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y3, Y10
	VPAND          Y0, Y3, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y4, Y0, Y10
	VPSRLQ         $0x04, Y4, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (R9)
	VMOVDQU        Y2, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y3, (R10)
	VMOVDQU        Y4, 32(R10)
	ADDQ           $0x40, R10
	VPSRLQ         $0x04, Y7, Y2
	VPAND          Y0, Y7, Y1
	VPAND          Y0, Y2, Y2
	VBROADCASTI128 (CX), Y3
	VBROADCASTI128 64(CX), Y4
	VPSHUFB        Y1, Y3, Y3
	VPSHUFB        Y1, Y4, Y1
	VBROADCASTI128 16(CX), Y4
	VBROADCASTI128 80(CX), Y9
	VPSHUFB        Y2, Y4, Y4
	VPSHUFB        Y2, Y9, Y2
	VPXOR          Y3, Y4, Y3
	VPXOR          Y1, Y2, Y1
	VPAND          Y8, Y0, Y2
	VPSRLQ         $0x04, Y8, Y4
	VPAND          Y0, Y4, Y4
	VBROADCASTI128 32(CX), Y9
	VBROADCASTI128 96(CX), Y10
	VPSHUFB        Y2, Y9, Y9
	VPSHUFB        Y2, Y10, Y2
	VPXOR          Y3, Y9, Y3
	VPXOR          Y1, Y2, Y1
	VBROADCASTI128 48(CX), Y9
	VBROADCASTI128 112(CX), Y2
	VPSHUFB        Y4, Y9, Y9
	VPSHUFB        Y4, Y2, Y2
	XOR3WAY(     $0x00, Y3, Y9, Y5)
	XOR3WAY(     $0x00, Y1, Y2, Y6)
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R11)
	VMOVDQU        Y6, 32(R11)
	ADDQ           $0x40, R11
	VMOVDQU        Y7, (BX)
	VMOVDQU        Y8, 32(BX)
	ADDQ           $0x40, BX
	SUBQ           $0x40, DI
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_1(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx2_1(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, DX
	MOVQ         DX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), DX
	MOVQ         work_base+0(FP), BX
	MOVQ         8(BX), SI
	XORQ         DI, DI
	MOVQ         (BX)(DI*1), R8
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R9
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R10
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), DX

loop:
	VMOVDQU        (R8), Y1
	VMOVDQU        32(R8), Y2
	VMOVDQU        (R9), Y3
	VMOVDQU        32(R9), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        (R10), Y5
	VMOVDQU        32(R10), Y6
	VMOVDQU        (DX), Y7
	VMOVDQU        32(DX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y5)
	XOR3WAY(     $0x00, Y9, Y10, Y6)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VMOVDQU        Y1, (R8)
	VMOVDQU        Y2, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y3, (R9)
	VMOVDQU        Y4, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y5, (R10)
	VMOVDQU        Y6, 32(R10)
	ADDQ           $0x40, R10
	VMOVDQU        Y7, (DX)
	VMOVDQU        Y8, 32(DX)
	ADDQ           $0x40, DX
	SUBQ           $0x40, SI
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_1(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx2_1(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), DX
	MOVQ         $0x0000000f, DX
	MOVQ         DX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), DX
	MOVQ         work_base+0(FP), BX
	MOVQ         8(BX), SI
	XORQ         DI, DI
	MOVQ         (BX)(DI*1), R8
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R9
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R10
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), DX

loop:
	VMOVDQU        (R8), Y1
	VMOVDQU        32(R8), Y2
	VMOVDQU        (R10), Y5
	VMOVDQU        32(R10), Y6
	VMOVDQU        (R9), Y3
	VMOVDQU        32(R9), Y4
	VMOVDQU        (DX), Y7
	VMOVDQU        32(DX), Y8
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y3, Y10
	VPAND          Y0, Y3, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y4, Y0, Y10
	VPSRLQ         $0x04, Y4, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (R8)
	VMOVDQU        Y2, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y3, (R9)
	VMOVDQU        Y4, 32(R9)
	ADDQ           $0x40, R9
	VPSRLQ         $0x04, Y7, Y2
	VPAND          Y0, Y7, Y1
	VPAND          Y0, Y2, Y2
	VBROADCASTI128 (CX), Y3
	VBROADCASTI128 64(CX), Y4
	VPSHUFB        Y1, Y3, Y3
	VPSHUFB        Y1, Y4, Y1
	VBROADCASTI128 16(CX), Y4
	VBROADCASTI128 80(CX), Y9
	VPSHUFB        Y2, Y4, Y4
	VPSHUFB        Y2, Y9, Y2
	VPXOR          Y3, Y4, Y3
	VPXOR          Y1, Y2, Y1
	VPAND          Y8, Y0, Y2
	VPSRLQ         $0x04, Y8, Y4
	VPAND          Y0, Y4, Y4
	VBROADCASTI128 32(CX), Y9
	VBROADCASTI128 96(CX), Y10
	VPSHUFB        Y2, Y9, Y9
	VPSHUFB        Y2, Y10, Y2
	VPXOR          Y3, Y9, Y3
	VPXOR          Y1, Y2, Y1
	VBROADCASTI128 48(CX), Y9
	VBROADCASTI128 112(CX), Y2
	VPSHUFB        Y4, Y9, Y9
	VPSHUFB        Y4, Y2, Y2
	XOR3WAY(     $0x00, Y3, Y9, Y5)
	XOR3WAY(     $0x00, Y1, Y2, Y6)
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R10)
	VMOVDQU        Y6, 32(R10)
	ADDQ           $0x40, R10
	VMOVDQU        Y7, (DX)
	VMOVDQU        Y8, 32(DX)
	ADDQ           $0x40, DX
	SUBQ           $0x40, SI
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_2(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx2_2(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, DX
	MOVQ         DX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), DX
	MOVQ         work_base+0(FP), BX
	MOVQ         8(BX), SI
	XORQ         DI, DI
	MOVQ         (BX)(DI*1), R8
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R9
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R10
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), DX

loop:
	VMOVDQU        (R8), Y1
	VMOVDQU        32(R8), Y2
	VMOVDQU        (R9), Y3
	VMOVDQU        32(R9), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VPSRLQ         $0x04, Y3, Y6
	VPAND          Y0, Y3, Y5
	VPAND          Y0, Y6, Y6
	VBROADCASTI128 (AX), Y7
	VBROADCASTI128 64(AX), Y8
	VPSHUFB        Y5, Y7, Y7
	VPSHUFB        Y5, Y8, Y5
	VBROADCASTI128 16(AX), Y8
	VBROADCASTI128 80(AX), Y9
	VPSHUFB        Y6, Y8, Y8
	VPSHUFB        Y6, Y9, Y6
	VPXOR          Y7, Y8, Y7
	VPXOR          Y5, Y6, Y5
	VPAND          Y4, Y0, Y6
	VPSRLQ         $0x04, Y4, Y8
	VPAND          Y0, Y8, Y8
	VBROADCASTI128 32(AX), Y9
	VBROADCASTI128 96(AX), Y10
	VPSHUFB        Y6, Y9, Y9
	VPSHUFB        Y6, Y10, Y6
	VPXOR          Y7, Y9, Y7
	VPXOR          Y5, Y6, Y5
	VBROADCASTI128 48(AX), Y9
	VBROADCASTI128 112(AX), Y6
	VPSHUFB        Y8, Y9, Y9
	VPSHUFB        Y8, Y6, Y6
	XOR3WAY(     $0x00, Y7, Y9, Y1)
	XOR3WAY(     $0x00, Y5, Y6, Y2)
	VMOVDQU        (R10), Y5
	VMOVDQU        32(R10), Y6
	VMOVDQU        (DX), Y7
	VMOVDQU        32(DX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VMOVDQU        Y1, (R8)
	VMOVDQU        Y2, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y3, (R9)
	VMOVDQU        Y4, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y5, (R10)
	VMOVDQU        Y6, 32(R10)
	ADDQ           $0x40, R10
	VMOVDQU        Y7, (DX)
	VMOVDQU        Y8, 32(DX)
	ADDQ           $0x40, DX
	SUBQ           $0x40, SI
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_2(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx2_2(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, DX
	MOVQ         DX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), DX
	MOVQ         work_base+0(FP), BX
	MOVQ         8(BX), SI
	XORQ         DI, DI
	MOVQ         (BX)(DI*1), R8
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R9
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R10
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), DX

loop:
	VMOVDQU        (R8), Y1
	VMOVDQU        32(R8), Y2
	VMOVDQU        (R10), Y5
	VMOVDQU        32(R10), Y6
	VMOVDQU        (R9), Y3
	VMOVDQU        32(R9), Y4
	VMOVDQU        (DX), Y7
	VMOVDQU        32(DX), Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (R8)
	VMOVDQU        Y2, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y3, (R9)
	VMOVDQU        Y4, 32(R9)
	ADDQ           $0x40, R9
	VPSRLQ         $0x04, Y7, Y2
	VPAND          Y0, Y7, Y1
	VPAND          Y0, Y2, Y2
	VBROADCASTI128 (AX), Y3
	VBROADCASTI128 64(AX), Y4
	VPSHUFB        Y1, Y3, Y3
	VPSHUFB        Y1, Y4, Y1
	VBROADCASTI128 16(AX), Y4
	VBROADCASTI128 80(AX), Y9
	VPSHUFB        Y2, Y4, Y4
	VPSHUFB        Y2, Y9, Y2
	VPXOR          Y3, Y4, Y3
	VPXOR          Y1, Y2, Y1
	VPAND          Y8, Y0, Y2
	VPSRLQ         $0x04, Y8, Y4
	VPAND          Y0, Y4, Y4
	VBROADCASTI128 32(AX), Y9
	VBROADCASTI128 96(AX), Y10
	VPSHUFB        Y2, Y9, Y9
	VPSHUFB        Y2, Y10, Y2
	VPXOR          Y3, Y9, Y3
	VPXOR          Y1, Y2, Y1
	VBROADCASTI128 48(AX), Y9
	VBROADCASTI128 112(AX), Y2
	VPSHUFB        Y4, Y9, Y9
	VPSHUFB        Y4, Y2, Y2
	XOR3WAY(     $0x00, Y3, Y9, Y5)
	XOR3WAY(     $0x00, Y1, Y2, Y6)
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R10)
	VMOVDQU        Y6, 32(R10)
	ADDQ           $0x40, R10
	VMOVDQU        Y7, (DX)
	VMOVDQU        Y8, 32(DX)
	ADDQ           $0x40, DX
	SUBQ           $0x40, SI
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_3(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx2_3(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), AX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), CX
	MOVQ         work_base+0(FP), DX
	MOVQ         8(DX), BX
	XORQ         SI, SI
	MOVQ         (DX)(SI*1), DI
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R8
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R9
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), CX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (CX), Y7
	VMOVDQU        32(CX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (CX)
	VMOVDQU        Y8, 32(CX)
	ADDQ           $0x40, CX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_3(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx2_3(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), CX
	MOVQ         work_base+0(FP), DX
	MOVQ         8(DX), BX
	XORQ         SI, SI
	MOVQ         (DX)(SI*1), DI
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R8
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R9
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), CX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VMOVDQU        (CX), Y7
	VMOVDQU        32(CX), Y8
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VPSRLQ         $0x04, Y7, Y2
	VPAND          Y0, Y7, Y1
	VPAND          Y0, Y2, Y2
	VBROADCASTI128 (AX), Y3
	VBROADCASTI128 64(AX), Y4
	VPSHUFB        Y1, Y3, Y3
	VPSHUFB        Y1, Y4, Y1
	VBROADCASTI128 16(AX), Y4
	VBROADCASTI128 80(AX), Y9
	VPSHUFB        Y2, Y4, Y4
	VPSHUFB        Y2, Y9, Y2
	VPXOR          Y3, Y4, Y3
	VPXOR          Y1, Y2, Y1
	VPAND          Y8, Y0, Y2
	VPSRLQ         $0x04, Y8, Y4
	VPAND          Y0, Y4, Y4
	VBROADCASTI128 32(AX), Y9
	VBROADCASTI128 96(AX), Y10
	VPSHUFB        Y2, Y9, Y9
	VPSHUFB        Y2, Y10, Y2
	VPXOR          Y3, Y9, Y3
	VPXOR          Y1, Y2, Y1
	VBROADCASTI128 48(AX), Y9
	VBROADCASTI128 112(AX), Y2
	VPSHUFB        Y4, Y9, Y9
	VPSHUFB        Y4, Y2, Y2
	XOR3WAY(     $0x00, Y3, Y9, Y5)
	XOR3WAY(     $0x00, Y1, Y2, Y6)
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (CX)
	VMOVDQU        Y8, 32(CX)
	ADDQ           $0x40, CX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_4(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx2_4(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), DX
	MOVQ         $0x0000000f, DX
	MOVQ         DX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), DX
	MOVQ         work_base+0(FP), BX
	MOVQ         8(BX), SI
	XORQ         DI, DI
	MOVQ         (BX)(DI*1), R8
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R9
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R10
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), DX

loop:
	VMOVDQU        (R8), Y1
	VMOVDQU        32(R8), Y2
	VMOVDQU        (R9), Y3
	VMOVDQU        32(R9), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VPSRLQ         $0x04, Y3, Y6
	VPAND          Y0, Y3, Y5
	VPAND          Y0, Y6, Y6
	VBROADCASTI128 (AX), Y7
	VBROADCASTI128 64(AX), Y8
	VPSHUFB        Y5, Y7, Y7
	VPSHUFB        Y5, Y8, Y5
	VBROADCASTI128 16(AX), Y8
	VBROADCASTI128 80(AX), Y9
	VPSHUFB        Y6, Y8, Y8
	VPSHUFB        Y6, Y9, Y6
	VPXOR          Y7, Y8, Y7
	VPXOR          Y5, Y6, Y5
	VPAND          Y4, Y0, Y6
	VPSRLQ         $0x04, Y4, Y8
	VPAND          Y0, Y8, Y8
	VBROADCASTI128 32(AX), Y9
	VBROADCASTI128 96(AX), Y10
	VPSHUFB        Y6, Y9, Y9
	VPSHUFB        Y6, Y10, Y6
	VPXOR          Y7, Y9, Y7
	VPXOR          Y5, Y6, Y5
	VBROADCASTI128 48(AX), Y9
	VBROADCASTI128 112(AX), Y6
	VPSHUFB        Y8, Y9, Y9
	VPSHUFB        Y8, Y6, Y6
	XOR3WAY(     $0x00, Y7, Y9, Y1)
	XOR3WAY(     $0x00, Y5, Y6, Y2)
	VMOVDQU        (R10), Y5
	VMOVDQU        32(R10), Y6
	VMOVDQU        (DX), Y7
	VMOVDQU        32(DX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y5)
	XOR3WAY(     $0x00, Y9, Y10, Y6)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VMOVDQU        Y1, (R8)
	VMOVDQU        Y2, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y3, (R9)
	VMOVDQU        Y4, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y5, (R10)
	VMOVDQU        Y6, 32(R10)
	ADDQ           $0x40, R10
	VMOVDQU        Y7, (DX)
	VMOVDQU        Y8, 32(DX)
	ADDQ           $0x40, DX
	SUBQ           $0x40, SI
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_4(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx2_4(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, DX
	MOVQ         DX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), DX
	MOVQ         work_base+0(FP), BX
	MOVQ         8(BX), SI
	XORQ         DI, DI
	MOVQ         (BX)(DI*1), R8
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R9
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), R10
	ADDQ         DX, DI
	MOVQ         (BX)(DI*1), DX

loop:
	VMOVDQU        (R8), Y1
	VMOVDQU        32(R8), Y2
	VMOVDQU        (R10), Y5
	VMOVDQU        32(R10), Y6
	VMOVDQU        (R9), Y3
	VMOVDQU        32(R9), Y4
	VMOVDQU        (DX), Y7
	VMOVDQU        32(DX), Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (CX), Y11
	VBROADCASTI128 64(CX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(CX), Y12
	VBROADCASTI128 80(CX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(CX), Y13
	VBROADCASTI128 96(CX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(CX), Y13
	VBROADCASTI128 112(CX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y3, Y10
	VPAND          Y0, Y3, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y4, Y0, Y10
	VPSRLQ         $0x04, Y4, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (R8)
	VMOVDQU        Y2, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y3, (R9)
	VMOVDQU        Y4, 32(R9)
	ADDQ           $0x40, R9
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R10)
	VMOVDQU        Y6, 32(R10)
	ADDQ           $0x40, R10
	VMOVDQU        Y7, (DX)
	VMOVDQU        Y8, 32(DX)
	ADDQ           $0x40, DX
	SUBQ           $0x40, SI
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_5(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx2_5(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), CX
	MOVQ         work_base+0(FP), DX
	MOVQ         8(DX), BX
	XORQ         SI, SI
	MOVQ         (DX)(SI*1), DI
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R8
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R9
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), CX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (CX), Y7
	VMOVDQU        32(CX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y5)
	XOR3WAY(     $0x00, Y9, Y10, Y6)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (CX)
	VMOVDQU        Y8, 32(CX)
	ADDQ           $0x40, CX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_5(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx2_5(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), CX
	MOVQ         work_base+0(FP), DX
	MOVQ         8(DX), BX
	XORQ         SI, SI
	MOVQ         (DX)(SI*1), DI
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R8
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R9
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), CX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VMOVDQU        (CX), Y7
	VMOVDQU        32(CX), Y8
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPSRLQ         $0x04, Y3, Y10
	VPAND          Y0, Y3, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y4, Y0, Y10
	VPSRLQ         $0x04, Y4, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (CX)
	VMOVDQU        Y8, 32(CX)
	ADDQ           $0x40, CX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_6(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT4_avx2_6(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), CX
	MOVQ         table02+48(FP), CX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), CX
	MOVQ         work_base+0(FP), DX
	MOVQ         8(DX), BX
	XORQ         SI, SI
	MOVQ         (DX)(SI*1), DI
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R8
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R9
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), CX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VPSRLQ         $0x04, Y3, Y6
	VPAND          Y0, Y3, Y5
	VPAND          Y0, Y6, Y6
	VBROADCASTI128 (AX), Y7
	VBROADCASTI128 64(AX), Y8
	VPSHUFB        Y5, Y7, Y7
	VPSHUFB        Y5, Y8, Y5
	VBROADCASTI128 16(AX), Y8
	VBROADCASTI128 80(AX), Y9
	VPSHUFB        Y6, Y8, Y8
	VPSHUFB        Y6, Y9, Y6
	VPXOR          Y7, Y8, Y7
	VPXOR          Y5, Y6, Y5
	VPAND          Y4, Y0, Y6
	VPSRLQ         $0x04, Y4, Y8
	VPAND          Y0, Y8, Y8
	VBROADCASTI128 32(AX), Y9
	VBROADCASTI128 96(AX), Y10
	VPSHUFB        Y6, Y9, Y9
	VPSHUFB        Y6, Y10, Y6
	VPXOR          Y7, Y9, Y7
	VPXOR          Y5, Y6, Y5
	VBROADCASTI128 48(AX), Y9
	VBROADCASTI128 112(AX), Y6
	VPSHUFB        Y8, Y9, Y9
	VPSHUFB        Y8, Y6, Y6
	XOR3WAY(     $0x00, Y7, Y9, Y1)
	XOR3WAY(     $0x00, Y5, Y6, Y2)
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (CX), Y7
	VMOVDQU        32(CX), Y8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (CX)
	VMOVDQU        Y8, 32(CX)
	ADDQ           $0x40, CX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_6(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT4_avx2_6(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), AX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), CX
	MOVQ         work_base+0(FP), DX
	MOVQ         8(DX), BX
	XORQ         SI, SI
	MOVQ         (DX)(SI*1), DI
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R8
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), R9
	ADDQ         CX, SI
	MOVQ         (DX)(SI*1), CX

loop:
	VMOVDQU        (DI), Y1
	VMOVDQU        32(DI), Y2
	VMOVDQU        (R9), Y5
	VMOVDQU        32(R9), Y6
	VMOVDQU        (R8), Y3
	VMOVDQU        32(R8), Y4
	VMOVDQU        (CX), Y7
	VMOVDQU        32(CX), Y8
	VPSRLQ         $0x04, Y5, Y10
	VPAND          Y0, Y5, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y6, Y0, Y10
	VPSRLQ         $0x04, Y6, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y1)
	XOR3WAY(     $0x00, Y9, Y10, Y2)
	VPSRLQ         $0x04, Y7, Y10
	VPAND          Y0, Y7, Y9
	VPAND          Y0, Y10, Y10
	VBROADCASTI128 (AX), Y11
	VBROADCASTI128 64(AX), Y12
	VPSHUFB        Y9, Y11, Y11
	VPSHUFB        Y9, Y12, Y9
	VBROADCASTI128 16(AX), Y12
	VBROADCASTI128 80(AX), Y13
	VPSHUFB        Y10, Y12, Y12
	VPSHUFB        Y10, Y13, Y10
	VPXOR          Y11, Y12, Y11
	VPXOR          Y9, Y10, Y9
	VPAND          Y8, Y0, Y10
	VPSRLQ         $0x04, Y8, Y12
	VPAND          Y0, Y12, Y12
	VBROADCASTI128 32(AX), Y13
	VBROADCASTI128 96(AX), Y14
	VPSHUFB        Y10, Y13, Y13
	VPSHUFB        Y10, Y14, Y10
	VPXOR          Y11, Y13, Y11
	VPXOR          Y9, Y10, Y9
	VBROADCASTI128 48(AX), Y13
	VBROADCASTI128 112(AX), Y10
	VPSHUFB        Y12, Y13, Y13
	VPSHUFB        Y12, Y10, Y10
	XOR3WAY(     $0x00, Y11, Y13, Y3)
	XOR3WAY(     $0x00, Y9, Y10, Y4)
	VPXOR          Y1, Y5, Y5
	VPXOR          Y2, Y6, Y6
	VPXOR          Y3, Y7, Y7
	VPXOR          Y4, Y8, Y8
	VPXOR          Y1, Y3, Y3
	VPXOR          Y2, Y4, Y4
	VMOVDQU        Y1, (DI)
	VMOVDQU        Y2, 32(DI)
	ADDQ           $0x40, DI
	VMOVDQU        Y3, (R8)
	VMOVDQU        Y4, 32(R8)
	ADDQ           $0x40, R8
	VPXOR          Y5, Y7, Y7
	VPXOR          Y6, Y8, Y8
	VMOVDQU        Y5, (R9)
	VMOVDQU        Y6, 32(R9)
	ADDQ           $0x40, R9
	VMOVDQU        Y7, (CX)
	VMOVDQU        Y8, 32(CX)
	ADDQ           $0x40, CX
	SUBQ           $0x40, BX
	JNZ            loop
	VZEROUPPER
	RET

// func ifftDIT4_avx2_7(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, SSE2
TEXT ·ifftDIT4_avx2_7(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), AX
	MOVQ         $0x0000000f, AX
	MOVQ         AX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), AX
	MOVQ         work_base+0(FP), CX
	MOVQ         8(CX), DX
	XORQ         BX, BX
	MOVQ         (CX)(BX*1), SI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), DI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), R8
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), AX

loop:
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y1
	VMOVDQU (DI), Y2
	VMOVDQU 32(DI), Y3
	VPXOR   Y0, Y2, Y2
	VPXOR   Y1, Y3, Y3
	VMOVDQU (R8), Y4
	VMOVDQU 32(R8), Y5
	VMOVDQU (AX), Y6
	VMOVDQU 32(AX), Y7
	VPXOR   Y4, Y6, Y6
	VPXOR   Y5, Y7, Y7
	VPXOR   Y0, Y4, Y4
	VPXOR   Y1, Y5, Y5
	VPXOR   Y2, Y6, Y6
	VPXOR   Y3, Y7, Y7
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y4, (R8)
	VMOVDQU Y5, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y6, (AX)
	VMOVDQU Y7, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JNZ     loop
	VZEROUPPER
	RET

// func fftDIT4_avx2_7(work [][]byte, dist int, table01 *[128]uint8, table23 *[128]uint8, table02 *[128]uint8)
// Requires: AVX, AVX2, SSE2
TEXT ·fftDIT4_avx2_7(SB), NOSPLIT, $0-56
	// dist must be multiplied by 24 (size of slice header)
	MOVQ         table01+32(FP), AX
	MOVQ         table23+40(FP), AX
	MOVQ         table02+48(FP), AX
	MOVQ         $0x0000000f, AX
	MOVQ         AX, X0
	VPBROADCASTB X0, Y0
	MOVQ         dist+24(FP), AX
	MOVQ         work_base+0(FP), CX
	MOVQ         8(CX), DX
	XORQ         BX, BX
	MOVQ         (CX)(BX*1), SI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), DI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), R8
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), AX

loop:
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y1
	VMOVDQU (R8), Y4
	VMOVDQU 32(R8), Y5
	VMOVDQU (DI), Y2
	VMOVDQU 32(DI), Y3
	VMOVDQU (AX), Y6
	VMOVDQU 32(AX), Y7
	VPXOR   Y0, Y4, Y4
	VPXOR   Y1, Y5, Y5
	VPXOR   Y2, Y6, Y6
	VPXOR   Y3, Y7, Y7
	VPXOR   Y0, Y2, Y2
	VPXOR   Y1, Y3, Y3
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VPXOR   Y4, Y6, Y6
	VPXOR   Y5, Y7, Y7
	VMOVDQU Y4, (R8)
	VMOVDQU Y5, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y6, (AX)
	VMOVDQU Y7, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JNZ     loop
	VZEROUPPER
	RET

// func ifftDIT2_ssse3(x []byte, y []byte, table *[128]uint8)
// Requires: SSE, SSE2, SSSE3
TEXT ·ifftDIT2_ssse3(SB), NOSPLIT, $0-56
	MOVQ   table+48(FP), AX
	MOVUPS (AX), X0
	MOVUPS 64(AX), X1
	MOVUPS 16(AX), X2
	MOVUPS 80(AX), X3
	MOVUPS 32(AX), X4
	MOVUPS 96(AX), X5
	XORPS  X6, X6
	MOVQ   $0x0000000f, CX
	MOVQ   CX, X7
	PSHUFB X6, X7
	MOVQ   x_len+8(FP), CX
	MOVQ   x_base+0(FP), DX
	MOVQ   y_base+24(FP), BX

loop:
	MOVUPS (DX), X6
	MOVUPS 32(DX), X8
	MOVUPS (BX), X9
	MOVUPS 32(BX), X10
	PXOR   X6, X9
	PXOR   X8, X10
	MOVUPS X9, (BX)
	MOVUPS X10, 32(BX)
	MOVAPS X9, X11
	PSRLQ  $0x04, X11
	MOVAPS X9, X9
	PAND   X7, X9
	PAND   X7, X11
	MOVUPS X0, X12
	MOVUPS X1, X13
	PSHUFB X9, X12
	PSHUFB X9, X13
	MOVUPS X2, X9
	MOVUPS X3, X14
	PSHUFB X11, X9
	PSHUFB X11, X14
	PXOR   X9, X12
	PXOR   X14, X13
	MOVAPS X10, X9
	MOVAPS X10, X10
	PAND   X7, X9
	PSRLQ  $0x04, X10
	PAND   X7, X10
	MOVUPS X4, X11
	MOVUPS X5, X14
	PSHUFB X9, X11
	PSHUFB X9, X14
	PXOR   X11, X12
	PXOR   X14, X13
	MOVUPS 48(AX), X11
	MOVUPS 112(AX), X14
	PSHUFB X10, X11
	PSHUFB X10, X14
	PXOR   X11, X12
	PXOR   X14, X13
	PXOR   X12, X6
	PXOR   X13, X8
	MOVUPS X6, (DX)
	MOVUPS X8, 32(DX)
	MOVUPS 16(DX), X6
	MOVUPS 48(DX), X8
	MOVUPS 16(BX), X9
	MOVUPS 48(BX), X10
	PXOR   X6, X9
	PXOR   X8, X10
	MOVUPS X9, 16(BX)
	MOVUPS X10, 48(BX)
	MOVAPS X9, X11
	PSRLQ  $0x04, X11
	MOVAPS X9, X9
	PAND   X7, X9
	PAND   X7, X11
	MOVUPS X0, X12
	MOVUPS X1, X13
	PSHUFB X9, X12
	PSHUFB X9, X13
	MOVUPS X2, X9
	MOVUPS X3, X14
	PSHUFB X11, X9
	PSHUFB X11, X14
	PXOR   X9, X12
	PXOR   X14, X13
	MOVAPS X10, X9
	MOVAPS X10, X10
	PAND   X7, X9
	PSRLQ  $0x04, X10
	PAND   X7, X10
	MOVUPS X4, X11
	MOVUPS X5, X14
	PSHUFB X9, X11
	PSHUFB X9, X14
	PXOR   X11, X12
	PXOR   X14, X13
	MOVUPS 48(AX), X11
	MOVUPS 112(AX), X14
	PSHUFB X10, X11
	PSHUFB X10, X14
	PXOR   X11, X12
	PXOR   X14, X13
	PXOR   X12, X6
	PXOR   X13, X8
	MOVUPS X6, 16(DX)
	MOVUPS X8, 48(DX)
	ADDQ   $0x40, DX
	ADDQ   $0x40, BX
	SUBQ   $0x40, CX
	JNZ    loop
	RET

// func fftDIT2_ssse3(x []byte, y []byte, table *[128]uint8)
// Requires: SSE, SSE2, SSSE3
TEXT ·fftDIT2_ssse3(SB), NOSPLIT, $0-56
	MOVQ   table+48(FP), AX
	MOVUPS (AX), X0
	MOVUPS 64(AX), X1
	MOVUPS 16(AX), X2
	MOVUPS 80(AX), X3
	MOVUPS 32(AX), X4
	MOVUPS 96(AX), X5
	XORPS  X6, X6
	MOVQ   $0x0000000f, CX
	MOVQ   CX, X7
	PSHUFB X6, X7
	MOVQ   x_len+8(FP), CX
	MOVQ   x_base+0(FP), DX
	MOVQ   y_base+24(FP), BX

loop:
	MOVUPS (BX), X9
	MOVUPS 32(BX), X10
	MOVAPS X9, X8
	PSRLQ  $0x04, X8
	MOVAPS X9, X6
	PAND   X7, X6
	PAND   X7, X8
	MOVUPS X0, X11
	MOVUPS X1, X12
	PSHUFB X6, X11
	PSHUFB X6, X12
	MOVUPS X2, X6
	MOVUPS X3, X13
	PSHUFB X8, X6
	PSHUFB X8, X13
	PXOR   X6, X11
	PXOR   X13, X12
	MOVAPS X10, X6
	MOVAPS X10, X8
	PAND   X7, X6
	PSRLQ  $0x04, X8
	PAND   X7, X8
	MOVUPS X4, X13
	MOVUPS X5, X14
	PSHUFB X6, X13
	PSHUFB X6, X14
	PXOR   X13, X11
	PXOR   X14, X12
	MOVUPS 48(AX), X13
	MOVUPS 112(AX), X14
	PSHUFB X8, X13
	PSHUFB X8, X14
	PXOR   X13, X11
	PXOR   X14, X12
	MOVUPS (DX), X6
	MOVUPS 32(DX), X8
	PXOR   X11, X6
	PXOR   X12, X8
	MOVUPS X6, (DX)
	MOVUPS X8, 32(DX)
	PXOR   X6, X9
	PXOR   X8, X10
	MOVUPS X9, (BX)
	MOVUPS X10, 32(BX)
	MOVUPS 16(BX), X9
	MOVUPS 48(BX), X10
	MOVAPS X9, X8
	PSRLQ  $0x04, X8
	MOVAPS X9, X6
	PAND   X7, X6
	PAND   X7, X8
	MOVUPS X0, X11
	MOVUPS X1, X12
	PSHUFB X6, X11
	PSHUFB X6, X12
	MOVUPS X2, X6
	MOVUPS X3, X13
	PSHUFB X8, X6
	PSHUFB X8, X13
	PXOR   X6, X11
	PXOR   X13, X12
	MOVAPS X10, X6
	MOVAPS X10, X8
	PAND   X7, X6
	PSRLQ  $0x04, X8
	PAND   X7, X8
	MOVUPS X4, X13
	MOVUPS X5, X14
	PSHUFB X6, X13
	PSHUFB X6, X14
	PXOR   X13, X11
	PXOR   X14, X12
	MOVUPS 48(AX), X13
	MOVUPS 112(AX), X14
	PSHUFB X8, X13
	PSHUFB X8, X14
	PXOR   X13, X11
	PXOR   X14, X12
	MOVUPS 16(DX), X6
	MOVUPS 48(DX), X8
	PXOR   X11, X6
	PXOR   X12, X8
	MOVUPS X6, 16(DX)
	MOVUPS X8, 48(DX)
	PXOR   X6, X9
	PXOR   X8, X10
	MOVUPS X9, 16(BX)
	MOVUPS X10, 48(BX)
	ADDQ   $0x40, DX
	ADDQ   $0x40, BX
	SUBQ   $0x40, CX
	JNZ    loop
	RET

// func mulgf16_ssse3(x []byte, y []byte, table *[128]uint8)
// Requires: SSE, SSE2, SSSE3
TEXT ·mulgf16_ssse3(SB), NOSPLIT, $0-56
	MOVQ   table+48(FP), AX
	MOVUPS (AX), X0
	MOVUPS 64(AX), X1
	MOVUPS 16(AX), X2
	MOVUPS 80(AX), X3
	MOVUPS 32(AX), X4
	MOVUPS 96(AX), X5
	MOVUPS 48(AX), X6
	MOVUPS 112(AX), X7
	MOVQ   x_len+8(FP), AX
	MOVQ   x_base+0(FP), CX
	MOVQ   y_base+24(FP), DX
	XORPS  X8, X8
	MOVQ   $0x0000000f, BX
	MOVQ   BX, X9
	PSHUFB X8, X9

loop:
	MOVUPS (DX), X8
	MOVUPS 32(DX), X10
	MOVAPS X8, X11
	PSRLQ  $0x04, X11
	MOVAPS X8, X8
	PAND   X9, X8
	PAND   X9, X11
	MOVUPS X0, X12
	MOVUPS X1, X13
	PSHUFB X8, X12
	PSHUFB X8, X13
	MOVUPS X2, X8
	MOVUPS X3, X14
	PSHUFB X11, X8
	PSHUFB X11, X14
	PXOR   X8, X12
	PXOR   X14, X13
	MOVAPS X10, X8
	MOVAPS X10, X10
	PAND   X9, X8
	PSRLQ  $0x04, X10
	PAND   X9, X10
	MOVUPS X4, X11
	MOVUPS X5, X14
	PSHUFB X8, X11
	PSHUFB X8, X14
	PXOR   X11, X12
	PXOR   X14, X13
	MOVUPS X6, X11
	MOVUPS X7, X14
	PSHUFB X10, X11
	PSHUFB X10, X14
	PXOR   X11, X12
	PXOR   X14, X13
	MOVUPS X12, (CX)
	MOVUPS X13, 32(CX)
	MOVUPS 16(DX), X8
	MOVUPS 48(DX), X10
	MOVAPS X8, X11
	PSRLQ  $0x04, X11
	MOVAPS X8, X8
	PAND   X9, X8
	PAND   X9, X11
	MOVUPS X0, X12
	MOVUPS X1, X13
	PSHUFB X8, X12
	PSHUFB X8, X13
	MOVUPS X2, X8
	MOVUPS X3, X14
	PSHUFB X11, X8
	PSHUFB X11, X14
	PXOR   X8, X12
	PXOR   X14, X13
	MOVAPS X10, X8
	MOVAPS X10, X10
	PAND   X9, X8
	PSRLQ  $0x04, X10
	PAND   X9, X10
	MOVUPS X4, X11
	MOVUPS X5, X14
	PSHUFB X8, X11
	PSHUFB X8, X14
	PXOR   X11, X12
	PXOR   X14, X13
	MOVUPS X6, X11
	MOVUPS X7, X14
	PSHUFB X10, X11
	PSHUFB X10, X14
	PXOR   X11, X12
	PXOR   X14, X13
	MOVUPS X12, 16(CX)
	MOVUPS X13, 48(CX)
	ADDQ   $0x40, CX
	ADDQ   $0x40, DX
	SUBQ   $0x40, AX
	JNZ    loop
	RET

// func ifftDIT28_avx2(x []byte, y []byte, table *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT28_avx2(SB), NOSPLIT, $0-56
	MOVQ           table+48(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           x_len+8(FP), AX
	MOVQ           x_base+0(FP), CX
	MOVQ           y_base+24(FP), DX
	MOVQ           $0x0000000f, BX
	MOVQ           BX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VMOVDQU (DX), Y5
	VMOVDQU 32(DX), Y6
	VPXOR   Y5, Y3, Y5
	VPXOR   Y6, Y4, Y6
	VMOVDQU Y5, (DX)
	VMOVDQU Y6, 32(DX)

	// LEO_MULADD_256
	VPAND   Y5, Y2, Y7
	VPSRLQ  $0x04, Y5, Y5
	VPSHUFB Y7, Y0, Y7
	VPAND   Y5, Y2, Y5
	VPSHUFB Y5, Y1, Y5
	XOR3WAY( $0x00, Y7, Y5, Y3)

	// LEO_MULADD_256
	VPAND   Y6, Y2, Y5
	VPSRLQ  $0x04, Y6, Y6
	VPSHUFB Y5, Y0, Y5
	VPAND   Y6, Y2, Y6
	VPSHUFB Y6, Y1, Y6
	XOR3WAY( $0x00, Y5, Y6, Y4)
	VMOVDQU Y3, (CX)
	VMOVDQU Y4, 32(CX)
	ADDQ    $0x40, CX
	ADDQ    $0x40, DX
	SUBQ    $0x40, AX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT28_avx2(x []byte, y []byte, table *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT28_avx2(SB), NOSPLIT, $0-56
	MOVQ           table+48(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           x_len+8(FP), AX
	MOVQ           x_base+0(FP), CX
	MOVQ           y_base+24(FP), DX
	MOVQ           $0x0000000f, BX
	MOVQ           BX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (CX), Y3
	VMOVDQU 32(CX), Y4
	VMOVDQU (DX), Y5
	VMOVDQU 32(DX), Y6

	// LEO_MULADD_256
	VPAND   Y5, Y2, Y7
	VPSRLQ  $0x04, Y5, Y8
	VPSHUFB Y7, Y0, Y7
	VPAND   Y8, Y2, Y8
	VPSHUFB Y8, Y1, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// LEO_MULADD_256
	VPAND   Y6, Y2, Y7
	VPSRLQ  $0x04, Y6, Y8
	VPSHUFB Y7, Y0, Y7
	VPAND   Y8, Y2, Y8
	VPSHUFB Y8, Y1, Y8
	XOR3WAY( $0x00, Y7, Y8, Y4)
	VMOVDQU Y3, (CX)
	VMOVDQU Y4, 32(CX)
	VPXOR   Y5, Y3, Y5
	VPXOR   Y6, Y4, Y6
	VMOVDQU Y5, (DX)
	VMOVDQU Y6, 32(DX)
	ADDQ    $0x40, CX
	ADDQ    $0x40, DX
	SUBQ    $0x40, AX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_0(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT48_avx2_0(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 16(AX), Y0
	MOVQ           t23+40(FP), CX
	VBROADCASTI128 (CX), Y1
	VBROADCASTI128 16(CX), Y2
	MOVQ           t02+48(FP), CX
	VBROADCASTI128 (CX), Y3
	VBROADCASTI128 16(CX), Y4
	MOVQ           dist+24(FP), CX
	MOVQ           work_base+0(FP), DX
	MOVQ           8(DX), BX
	XORQ           SI, SI
	MOVQ           (DX)(SI*1), DI
	ADDQ           CX, SI
	MOVQ           (DX)(SI*1), R8
	ADDQ           CX, SI
	MOVQ           (DX)(SI*1), R9
	ADDQ           CX, SI
	MOVQ           (DX)(SI*1), CX
	MOVQ           $0x0000000f, DX
	MOVQ           DX, X5
	VPBROADCASTB   X5, Y5

loop:
	VMOVDQU        (DI), Y6
	VMOVDQU        (R8), Y7
	VMOVDQU        32(DI), Y8
	VMOVDQU        32(R8), Y9
	VPXOR          Y7, Y6, Y7
	VPXOR          Y9, Y8, Y9
	VBROADCASTI128 (AX), Y10

	// LEO_MULADD_256
	VPAND   Y7, Y5, Y11
	VPSRLQ  $0x04, Y7, Y12
	VPSHUFB Y11, Y10, Y11
	VPAND   Y12, Y5, Y12
	VPSHUFB Y12, Y0, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)

	// LEO_MULADD_256
	VPAND   Y9, Y5, Y11
	VPSRLQ  $0x04, Y9, Y12
	VPSHUFB Y11, Y10, Y11
	VPAND   Y12, Y5, Y12
	VPSHUFB Y12, Y0, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VMOVDQU (R9), Y10
	VMOVDQU (CX), Y11
	VMOVDQU 32(R9), Y12
	VMOVDQU 32(CX), Y13
	VPXOR   Y10, Y11, Y11
	VPXOR   Y12, Y13, Y13

	// LEO_MULADD_256
	VPAND   Y11, Y5, Y14
	VPSRLQ  $0x04, Y11, Y15
	VPSHUFB Y14, Y1, Y14
	VPAND   Y15, Y5, Y15
	VPSHUFB Y15, Y2, Y15
	XOR3WAY( $0x00, Y14, Y15, Y10)

	// LEO_MULADD_256
	VPAND   Y13, Y5, Y14
	VPSRLQ  $0x04, Y13, Y15
	VPSHUFB Y14, Y1, Y14
	VPAND   Y15, Y5, Y15
	VPSHUFB Y15, Y2, Y15
	XOR3WAY( $0x00, Y14, Y15, Y12)
	VPXOR   Y6, Y10, Y10
	VPXOR   Y7, Y11, Y11
	VPXOR   Y8, Y12, Y12
	VPXOR   Y9, Y13, Y13

	// LEO_MULADD_256
	VPAND   Y10, Y5, Y14
	VPSRLQ  $0x04, Y10, Y15
	VPSHUFB Y14, Y3, Y14
	VPAND   Y15, Y5, Y15
	VPSHUFB Y15, Y4, Y15
	XOR3WAY( $0x00, Y14, Y15, Y6)

	// LEO_MULADD_256
	VPAND   Y11, Y5, Y14
	VPSRLQ  $0x04, Y11, Y15
	VPSHUFB Y14, Y3, Y14
	VPAND   Y15, Y5, Y15
	VPSHUFB Y15, Y4, Y15
	XOR3WAY( $0x00, Y14, Y15, Y7)

	// LEO_MULADD_256
	VPAND   Y12, Y5, Y14
	VPSRLQ  $0x04, Y12, Y15
	VPSHUFB Y14, Y3, Y14
	VPAND   Y15, Y5, Y15
	VPSHUFB Y15, Y4, Y15
	XOR3WAY( $0x00, Y14, Y15, Y8)

	// LEO_MULADD_256
	VPAND   Y13, Y5, Y14
	VPSRLQ  $0x04, Y13, Y15
	VPSHUFB Y14, Y3, Y14
	VPAND   Y15, Y5, Y15
	VPSHUFB Y15, Y4, Y15
	XOR3WAY( $0x00, Y14, Y15, Y9)
	VMOVDQU Y6, (DI)
	VMOVDQU Y8, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y7, (R8)
	VMOVDQU Y9, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y10, (R9)
	VMOVDQU Y12, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y11, (CX)
	VMOVDQU Y13, 32(CX)
	ADDQ    $0x40, CX
	SUBQ    $0x40, BX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_0(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT48_avx2_0(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 16(AX), Y0
	MOVQ           t23+40(FP), CX
	VBROADCASTI128 16(CX), Y1
	MOVQ           t02+48(FP), DX
	VBROADCASTI128 (DX), Y2
	VBROADCASTI128 16(DX), Y3
	MOVQ           dist+24(FP), DX
	MOVQ           work_base+0(FP), BX
	MOVQ           8(BX), SI
	XORQ           DI, DI
	MOVQ           (BX)(DI*1), R8
	ADDQ           DX, DI
	MOVQ           (BX)(DI*1), R9
	ADDQ           DX, DI
	MOVQ           (BX)(DI*1), R10
	ADDQ           DX, DI
	MOVQ           (BX)(DI*1), DX
	MOVQ           $0x0000000f, BX
	MOVQ           BX, X4
	VPBROADCASTB   X4, Y4

loop:
	VMOVDQU (R8), Y5
	VMOVDQU 32(R8), Y6
	VMOVDQU (R10), Y9
	VMOVDQU 32(R10), Y10
	VMOVDQU (R9), Y7
	VMOVDQU 32(R9), Y8
	VMOVDQU (DX), Y11
	VMOVDQU 32(DX), Y12

	// LEO_MULADD_256
	VPAND   Y9, Y4, Y13
	VPSRLQ  $0x04, Y9, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y5)

	// LEO_MULADD_256
	VPAND   Y10, Y4, Y13
	VPSRLQ  $0x04, Y10, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y6)

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y13
	VPSRLQ  $0x04, Y11, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y7)

	// LEO_MULADD_256
	VPAND          Y12, Y4, Y13
	VPSRLQ         $0x04, Y12, Y14
	VPSHUFB        Y13, Y2, Y13
	VPAND          Y14, Y4, Y14
	VPSHUFB        Y14, Y3, Y14
	XOR3WAY(     $0x00, Y13, Y14, Y8)
	VPXOR          Y5, Y9, Y9
	VPXOR          Y7, Y11, Y11
	VPXOR          Y6, Y10, Y10
	VPXOR          Y8, Y12, Y12
	VBROADCASTI128 (AX), Y13

	// LEO_MULADD_256
	VPAND   Y7, Y4, Y14
	VPSRLQ  $0x04, Y7, Y15
	VPSHUFB Y14, Y13, Y14
	VPAND   Y15, Y4, Y15
	VPSHUFB Y15, Y0, Y15
	XOR3WAY( $0x00, Y14, Y15, Y5)

	// LEO_MULADD_256
	VPAND          Y8, Y4, Y14
	VPSRLQ         $0x04, Y8, Y15
	VPSHUFB        Y14, Y13, Y14
	VPAND          Y15, Y4, Y15
	VPSHUFB        Y15, Y0, Y15
	XOR3WAY(     $0x00, Y14, Y15, Y6)
	VPXOR          Y7, Y5, Y7
	VPXOR          Y8, Y6, Y8
	VBROADCASTI128 (CX), Y13

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y14
	VPSRLQ  $0x04, Y11, Y15
	VPSHUFB Y14, Y13, Y14
	VPAND   Y15, Y4, Y15
	VPSHUFB Y15, Y1, Y15
	XOR3WAY( $0x00, Y14, Y15, Y9)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y14
	VPSRLQ  $0x04, Y12, Y15
	VPSHUFB Y14, Y13, Y14
	VPAND   Y15, Y4, Y15
	VPSHUFB Y15, Y1, Y15
	XOR3WAY( $0x00, Y14, Y15, Y10)
	VPXOR   Y9, Y11, Y11
	VPXOR   Y10, Y12, Y12
	VMOVDQU Y5, (R8)
	VMOVDQU Y6, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y7, (R9)
	VMOVDQU Y8, 32(R9)
	ADDQ    $0x40, R9
	VMOVDQU Y9, (R10)
	VMOVDQU Y10, 32(R10)
	ADDQ    $0x40, R10
	VMOVDQU Y11, (DX)
	VMOVDQU Y12, 32(DX)
	ADDQ    $0x40, DX
	SUBQ    $0x40, SI
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_1(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT48_avx2_1(SB), NOSPLIT, $0-56
	MOVQ           t23+40(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           t02+48(FP), AX
	VBROADCASTI128 (AX), Y2
	VBROADCASTI128 16(AX), Y3
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X4
	VPBROADCASTB   X4, Y4

loop:
	VMOVDQU (SI), Y5
	VMOVDQU (DI), Y6
	VMOVDQU 32(SI), Y7
	VMOVDQU 32(DI), Y8
	VPXOR   Y6, Y5, Y6
	VPXOR   Y8, Y7, Y8
	VMOVDQU (R8), Y9
	VMOVDQU (AX), Y10
	VMOVDQU 32(R8), Y11
	VMOVDQU 32(AX), Y12
	VPXOR   Y9, Y10, Y10
	VPXOR   Y11, Y12, Y12

	// LEO_MULADD_256
	VPAND   Y10, Y4, Y13
	VPSRLQ  $0x04, Y10, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y9)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y11)
	VPXOR   Y5, Y9, Y9
	VPXOR   Y6, Y10, Y10
	VPXOR   Y7, Y11, Y11
	VPXOR   Y8, Y12, Y12

	// LEO_MULADD_256
	VPAND   Y9, Y4, Y13
	VPSRLQ  $0x04, Y9, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y5)

	// LEO_MULADD_256
	VPAND   Y10, Y4, Y13
	VPSRLQ  $0x04, Y10, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y6)

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y13
	VPSRLQ  $0x04, Y11, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y7)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y8)
	VMOVDQU Y5, (SI)
	VMOVDQU Y7, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y6, (DI)
	VMOVDQU Y8, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y9, (R8)
	VMOVDQU Y11, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y10, (AX)
	VMOVDQU Y12, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_1(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT48_avx2_1(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           t23+40(FP), AX
	VBROADCASTI128 (AX), Y2
	VBROADCASTI128 16(AX), Y3
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X4
	VPBROADCASTB   X4, Y4

loop:
	VMOVDQU (SI), Y5
	VMOVDQU 32(SI), Y6
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y10
	VMOVDQU (DI), Y7
	VMOVDQU 32(DI), Y8
	VMOVDQU (AX), Y11
	VMOVDQU 32(AX), Y12
	VPXOR   Y5, Y9, Y9
	VPXOR   Y7, Y11, Y11
	VPXOR   Y6, Y10, Y10
	VPXOR   Y8, Y12, Y12

	// LEO_MULADD_256
	VPAND   Y7, Y4, Y13
	VPSRLQ  $0x04, Y7, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y5)

	// LEO_MULADD_256
	VPAND   Y8, Y4, Y13
	VPSRLQ  $0x04, Y8, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y6)
	VPXOR   Y7, Y5, Y7
	VPXOR   Y8, Y6, Y8

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y13
	VPSRLQ  $0x04, Y11, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y9)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y10)
	VPXOR   Y9, Y11, Y11
	VPXOR   Y10, Y12, Y12
	VMOVDQU Y5, (SI)
	VMOVDQU Y6, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y7, (DI)
	VMOVDQU Y8, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y9, (R8)
	VMOVDQU Y10, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y11, (AX)
	VMOVDQU Y12, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_2(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT48_avx2_2(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           t02+48(FP), AX
	VBROADCASTI128 (AX), Y2
	VBROADCASTI128 16(AX), Y3
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X4
	VPBROADCASTB   X4, Y4

loop:
	VMOVDQU (SI), Y5
	VMOVDQU (DI), Y6
	VMOVDQU 32(SI), Y7
	VMOVDQU 32(DI), Y8
	VPXOR   Y6, Y5, Y6
	VPXOR   Y8, Y7, Y8

	// LEO_MULADD_256
	VPAND   Y6, Y4, Y9
	VPSRLQ  $0x04, Y6, Y10
	VPSHUFB Y9, Y0, Y9
	VPAND   Y10, Y4, Y10
	VPSHUFB Y10, Y1, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// LEO_MULADD_256
	VPAND   Y8, Y4, Y9
	VPSRLQ  $0x04, Y8, Y10
	VPSHUFB Y9, Y0, Y9
	VPAND   Y10, Y4, Y10
	VPSHUFB Y10, Y1, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)
	VMOVDQU (R8), Y9
	VMOVDQU (AX), Y10
	VMOVDQU 32(R8), Y11
	VMOVDQU 32(AX), Y12
	VPXOR   Y9, Y10, Y10
	VPXOR   Y11, Y12, Y12
	VPXOR   Y5, Y9, Y9
	VPXOR   Y6, Y10, Y10
	VPXOR   Y7, Y11, Y11
	VPXOR   Y8, Y12, Y12

	// LEO_MULADD_256
	VPAND   Y9, Y4, Y13
	VPSRLQ  $0x04, Y9, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y5)

	// LEO_MULADD_256
	VPAND   Y10, Y4, Y13
	VPSRLQ  $0x04, Y10, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y6)

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y13
	VPSRLQ  $0x04, Y11, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y7)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y8)
	VMOVDQU Y5, (SI)
	VMOVDQU Y7, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y6, (DI)
	VMOVDQU Y8, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y9, (R8)
	VMOVDQU Y11, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y10, (AX)
	VMOVDQU Y12, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_2(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT48_avx2_2(SB), NOSPLIT, $0-56
	MOVQ           t23+40(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           t02+48(FP), AX
	VBROADCASTI128 (AX), Y2
	VBROADCASTI128 16(AX), Y3
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X4
	VPBROADCASTB   X4, Y4

loop:
	VMOVDQU (SI), Y5
	VMOVDQU 32(SI), Y6
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y10
	VMOVDQU (DI), Y7
	VMOVDQU 32(DI), Y8
	VMOVDQU (AX), Y11
	VMOVDQU 32(AX), Y12

	// LEO_MULADD_256
	VPAND   Y9, Y4, Y13
	VPSRLQ  $0x04, Y9, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y5)

	// LEO_MULADD_256
	VPAND   Y10, Y4, Y13
	VPSRLQ  $0x04, Y10, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y6)

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y13
	VPSRLQ  $0x04, Y11, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y7)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y8)
	VPXOR   Y5, Y9, Y9
	VPXOR   Y7, Y11, Y11
	VPXOR   Y6, Y10, Y10
	VPXOR   Y8, Y12, Y12
	VPXOR   Y7, Y5, Y7
	VPXOR   Y8, Y6, Y8

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y13
	VPSRLQ  $0x04, Y11, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y9)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y10)
	VPXOR   Y9, Y11, Y11
	VPXOR   Y10, Y12, Y12
	VMOVDQU Y5, (SI)
	VMOVDQU Y6, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y7, (DI)
	VMOVDQU Y8, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y9, (R8)
	VMOVDQU Y10, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y11, (AX)
	VMOVDQU Y12, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_3(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT48_avx2_3(SB), NOSPLIT, $0-56
	MOVQ           t02+48(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (SI), Y3
	VMOVDQU (DI), Y4
	VMOVDQU 32(SI), Y5
	VMOVDQU 32(DI), Y6
	VPXOR   Y4, Y3, Y4
	VPXOR   Y6, Y5, Y6
	VMOVDQU (R8), Y7
	VMOVDQU (AX), Y8
	VMOVDQU 32(R8), Y9
	VMOVDQU 32(AX), Y10
	VPXOR   Y7, Y8, Y8
	VPXOR   Y9, Y10, Y10
	VPXOR   Y3, Y7, Y7
	VPXOR   Y4, Y8, Y8
	VPXOR   Y5, Y9, Y9
	VPXOR   Y6, Y10, Y10

	// LEO_MULADD_256
	VPAND   Y7, Y2, Y11
	VPSRLQ  $0x04, Y7, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)

	// LEO_MULADD_256
	VPAND   Y8, Y2, Y11
	VPSRLQ  $0x04, Y8, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)

	// LEO_MULADD_256
	VPAND   Y9, Y2, Y11
	VPSRLQ  $0x04, Y9, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)

	// LEO_MULADD_256
	VPAND   Y10, Y2, Y11
	VPSRLQ  $0x04, Y10, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VMOVDQU Y3, (SI)
	VMOVDQU Y5, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y4, (DI)
	VMOVDQU Y6, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y7, (R8)
	VMOVDQU Y9, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y8, (AX)
	VMOVDQU Y10, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_3(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT48_avx2_3(SB), NOSPLIT, $0-56
	MOVQ           t23+40(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (SI), Y3
	VMOVDQU 32(SI), Y4
	VMOVDQU (R8), Y7
	VMOVDQU 32(R8), Y8
	VMOVDQU (DI), Y5
	VMOVDQU 32(DI), Y6
	VMOVDQU (AX), Y9
	VMOVDQU 32(AX), Y10
	VPXOR   Y3, Y7, Y7
	VPXOR   Y5, Y9, Y9
	VPXOR   Y4, Y8, Y8
	VPXOR   Y6, Y10, Y10
	VPXOR   Y5, Y3, Y5
	VPXOR   Y6, Y4, Y6

	// LEO_MULADD_256
	VPAND   Y9, Y2, Y11
	VPSRLQ  $0x04, Y9, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)

	// LEO_MULADD_256
	VPAND   Y10, Y2, Y11
	VPSRLQ  $0x04, Y10, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y8)
	VPXOR   Y7, Y9, Y9
	VPXOR   Y8, Y10, Y10
	VMOVDQU Y3, (SI)
	VMOVDQU Y4, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y5, (DI)
	VMOVDQU Y6, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y7, (R8)
	VMOVDQU Y8, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y9, (AX)
	VMOVDQU Y10, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_4(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT48_avx2_4(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           t23+40(FP), AX
	VBROADCASTI128 (AX), Y2
	VBROADCASTI128 16(AX), Y3
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X4
	VPBROADCASTB   X4, Y4

loop:
	VMOVDQU (SI), Y5
	VMOVDQU (DI), Y6
	VMOVDQU 32(SI), Y7
	VMOVDQU 32(DI), Y8
	VPXOR   Y6, Y5, Y6
	VPXOR   Y8, Y7, Y8

	// LEO_MULADD_256
	VPAND   Y6, Y4, Y9
	VPSRLQ  $0x04, Y6, Y10
	VPSHUFB Y9, Y0, Y9
	VPAND   Y10, Y4, Y10
	VPSHUFB Y10, Y1, Y10
	XOR3WAY( $0x00, Y9, Y10, Y5)

	// LEO_MULADD_256
	VPAND   Y8, Y4, Y9
	VPSRLQ  $0x04, Y8, Y10
	VPSHUFB Y9, Y0, Y9
	VPAND   Y10, Y4, Y10
	VPSHUFB Y10, Y1, Y10
	XOR3WAY( $0x00, Y9, Y10, Y7)
	VMOVDQU (R8), Y9
	VMOVDQU (AX), Y10
	VMOVDQU 32(R8), Y11
	VMOVDQU 32(AX), Y12
	VPXOR   Y9, Y10, Y10
	VPXOR   Y11, Y12, Y12

	// LEO_MULADD_256
	VPAND   Y10, Y4, Y13
	VPSRLQ  $0x04, Y10, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y9)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y11)
	VPXOR   Y5, Y9, Y9
	VPXOR   Y6, Y10, Y10
	VPXOR   Y7, Y11, Y11
	VPXOR   Y8, Y12, Y12
	VMOVDQU Y5, (SI)
	VMOVDQU Y7, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y6, (DI)
	VMOVDQU Y8, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y9, (R8)
	VMOVDQU Y11, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y10, (AX)
	VMOVDQU Y12, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_4(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT48_avx2_4(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           t02+48(FP), AX
	VBROADCASTI128 (AX), Y2
	VBROADCASTI128 16(AX), Y3
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X4
	VPBROADCASTB   X4, Y4

loop:
	VMOVDQU (SI), Y5
	VMOVDQU 32(SI), Y6
	VMOVDQU (R8), Y9
	VMOVDQU 32(R8), Y10
	VMOVDQU (DI), Y7
	VMOVDQU 32(DI), Y8
	VMOVDQU (AX), Y11
	VMOVDQU 32(AX), Y12

	// LEO_MULADD_256
	VPAND   Y9, Y4, Y13
	VPSRLQ  $0x04, Y9, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y5)

	// LEO_MULADD_256
	VPAND   Y10, Y4, Y13
	VPSRLQ  $0x04, Y10, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y6)

	// LEO_MULADD_256
	VPAND   Y11, Y4, Y13
	VPSRLQ  $0x04, Y11, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y7)

	// LEO_MULADD_256
	VPAND   Y12, Y4, Y13
	VPSRLQ  $0x04, Y12, Y14
	VPSHUFB Y13, Y2, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y3, Y14
	XOR3WAY( $0x00, Y13, Y14, Y8)
	VPXOR   Y5, Y9, Y9
	VPXOR   Y7, Y11, Y11
	VPXOR   Y6, Y10, Y10
	VPXOR   Y8, Y12, Y12

	// LEO_MULADD_256
	VPAND   Y7, Y4, Y13
	VPSRLQ  $0x04, Y7, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y5)

	// LEO_MULADD_256
	VPAND   Y8, Y4, Y13
	VPSRLQ  $0x04, Y8, Y14
	VPSHUFB Y13, Y0, Y13
	VPAND   Y14, Y4, Y14
	VPSHUFB Y14, Y1, Y14
	XOR3WAY( $0x00, Y13, Y14, Y6)
	VPXOR   Y7, Y5, Y7
	VPXOR   Y8, Y6, Y8
	VPXOR   Y9, Y11, Y11
	VPXOR   Y10, Y12, Y12
	VMOVDQU Y5, (SI)
	VMOVDQU Y6, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y7, (DI)
	VMOVDQU Y8, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y9, (R8)
	VMOVDQU Y10, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y11, (AX)
	VMOVDQU Y12, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_5(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT48_avx2_5(SB), NOSPLIT, $0-56
	MOVQ           t23+40(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (SI), Y3
	VMOVDQU (DI), Y4
	VMOVDQU 32(SI), Y5
	VMOVDQU 32(DI), Y6
	VPXOR   Y4, Y3, Y4
	VPXOR   Y6, Y5, Y6
	VMOVDQU (R8), Y7
	VMOVDQU (AX), Y8
	VMOVDQU 32(R8), Y9
	VMOVDQU 32(AX), Y10
	VPXOR   Y7, Y8, Y8
	VPXOR   Y9, Y10, Y10

	// LEO_MULADD_256
	VPAND   Y8, Y2, Y11
	VPSRLQ  $0x04, Y8, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y7)

	// LEO_MULADD_256
	VPAND   Y10, Y2, Y11
	VPSRLQ  $0x04, Y10, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y9)
	VPXOR   Y3, Y7, Y7
	VPXOR   Y4, Y8, Y8
	VPXOR   Y5, Y9, Y9
	VPXOR   Y6, Y10, Y10
	VMOVDQU Y3, (SI)
	VMOVDQU Y5, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y4, (DI)
	VMOVDQU Y6, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y7, (R8)
	VMOVDQU Y9, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y8, (AX)
	VMOVDQU Y10, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_5(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT48_avx2_5(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (SI), Y3
	VMOVDQU 32(SI), Y4
	VMOVDQU (R8), Y7
	VMOVDQU 32(R8), Y8
	VMOVDQU (DI), Y5
	VMOVDQU 32(DI), Y6
	VMOVDQU (AX), Y9
	VMOVDQU 32(AX), Y10
	VPXOR   Y3, Y7, Y7
	VPXOR   Y5, Y9, Y9
	VPXOR   Y4, Y8, Y8
	VPXOR   Y6, Y10, Y10

	// LEO_MULADD_256
	VPAND   Y5, Y2, Y11
	VPSRLQ  $0x04, Y5, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)

	// LEO_MULADD_256
	VPAND   Y6, Y2, Y11
	VPSRLQ  $0x04, Y6, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)
	VPXOR   Y5, Y3, Y5
	VPXOR   Y6, Y4, Y6
	VPXOR   Y7, Y9, Y9
	VPXOR   Y8, Y10, Y10
	VMOVDQU Y3, (SI)
	VMOVDQU Y4, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y5, (DI)
	VMOVDQU Y6, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y7, (R8)
	VMOVDQU Y8, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y9, (AX)
	VMOVDQU Y10, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_6(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·ifftDIT48_avx2_6(SB), NOSPLIT, $0-56
	MOVQ           t01+32(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (SI), Y3
	VMOVDQU (DI), Y4
	VMOVDQU 32(SI), Y5
	VMOVDQU 32(DI), Y6
	VPXOR   Y4, Y3, Y4
	VPXOR   Y6, Y5, Y6

	// LEO_MULADD_256
	VPAND   Y4, Y2, Y7
	VPSRLQ  $0x04, Y4, Y8
	VPSHUFB Y7, Y0, Y7
	VPAND   Y8, Y2, Y8
	VPSHUFB Y8, Y1, Y8
	XOR3WAY( $0x00, Y7, Y8, Y3)

	// LEO_MULADD_256
	VPAND   Y6, Y2, Y7
	VPSRLQ  $0x04, Y6, Y8
	VPSHUFB Y7, Y0, Y7
	VPAND   Y8, Y2, Y8
	VPSHUFB Y8, Y1, Y8
	XOR3WAY( $0x00, Y7, Y8, Y5)
	VMOVDQU (R8), Y7
	VMOVDQU (AX), Y8
	VMOVDQU 32(R8), Y9
	VMOVDQU 32(AX), Y10
	VPXOR   Y7, Y8, Y8
	VPXOR   Y9, Y10, Y10
	VPXOR   Y3, Y7, Y7
	VPXOR   Y4, Y8, Y8
	VPXOR   Y5, Y9, Y9
	VPXOR   Y6, Y10, Y10
	VMOVDQU Y3, (SI)
	VMOVDQU Y5, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y4, (DI)
	VMOVDQU Y6, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y7, (R8)
	VMOVDQU Y9, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y8, (AX)
	VMOVDQU Y10, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_6(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, AVX512F, AVX512VL, SSE2
TEXT ·fftDIT48_avx2_6(SB), NOSPLIT, $0-56
	MOVQ           t02+48(FP), AX
	VBROADCASTI128 (AX), Y0
	VBROADCASTI128 16(AX), Y1
	MOVQ           dist+24(FP), AX
	MOVQ           work_base+0(FP), CX
	MOVQ           8(CX), DX
	XORQ           BX, BX
	MOVQ           (CX)(BX*1), SI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), DI
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), R8
	ADDQ           AX, BX
	MOVQ           (CX)(BX*1), AX
	MOVQ           $0x0000000f, CX
	MOVQ           CX, X2
	VPBROADCASTB   X2, Y2

loop:
	VMOVDQU (SI), Y3
	VMOVDQU 32(SI), Y4
	VMOVDQU (R8), Y7
	VMOVDQU 32(R8), Y8
	VMOVDQU (DI), Y5
	VMOVDQU 32(DI), Y6
	VMOVDQU (AX), Y9
	VMOVDQU 32(AX), Y10

	// LEO_MULADD_256
	VPAND   Y7, Y2, Y11
	VPSRLQ  $0x04, Y7, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y3)

	// LEO_MULADD_256
	VPAND   Y8, Y2, Y11
	VPSRLQ  $0x04, Y8, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y4)

	// LEO_MULADD_256
	VPAND   Y9, Y2, Y11
	VPSRLQ  $0x04, Y9, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y5)

	// LEO_MULADD_256
	VPAND   Y10, Y2, Y11
	VPSRLQ  $0x04, Y10, Y12
	VPSHUFB Y11, Y0, Y11
	VPAND   Y12, Y2, Y12
	VPSHUFB Y12, Y1, Y12
	XOR3WAY( $0x00, Y11, Y12, Y6)
	VPXOR   Y3, Y7, Y7
	VPXOR   Y5, Y9, Y9
	VPXOR   Y4, Y8, Y8
	VPXOR   Y6, Y10, Y10
	VPXOR   Y5, Y3, Y5
	VPXOR   Y6, Y4, Y6
	VPXOR   Y7, Y9, Y9
	VPXOR   Y8, Y10, Y10
	VMOVDQU Y3, (SI)
	VMOVDQU Y4, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y5, (DI)
	VMOVDQU Y6, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y7, (R8)
	VMOVDQU Y8, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y9, (AX)
	VMOVDQU Y10, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_avx2_7(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, SSE2
TEXT ·ifftDIT48_avx2_7(SB), NOSPLIT, $0-56
	MOVQ         dist+24(FP), AX
	MOVQ         work_base+0(FP), CX
	MOVQ         8(CX), DX
	XORQ         BX, BX
	MOVQ         (CX)(BX*1), SI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), DI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), R8
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), AX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0

loop:
	VMOVDQU (SI), Y0
	VMOVDQU (DI), Y1
	VMOVDQU 32(SI), Y2
	VMOVDQU 32(DI), Y3
	VPXOR   Y1, Y0, Y1
	VPXOR   Y3, Y2, Y3
	VMOVDQU (R8), Y4
	VMOVDQU (AX), Y5
	VMOVDQU 32(R8), Y6
	VMOVDQU 32(AX), Y7
	VPXOR   Y4, Y5, Y5
	VPXOR   Y6, Y7, Y7
	VPXOR   Y0, Y4, Y4
	VPXOR   Y1, Y5, Y5
	VPXOR   Y2, Y6, Y6
	VPXOR   Y3, Y7, Y7
	VMOVDQU Y0, (SI)
	VMOVDQU Y2, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y1, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y4, (R8)
	VMOVDQU Y6, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y5, (AX)
	VMOVDQU Y7, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func fftDIT48_avx2_7(work [][]byte, dist int, t01 *[32]uint8, t23 *[32]uint8, t02 *[32]uint8)
// Requires: AVX, AVX2, SSE2
TEXT ·fftDIT48_avx2_7(SB), NOSPLIT, $0-56
	MOVQ         dist+24(FP), AX
	MOVQ         work_base+0(FP), CX
	MOVQ         8(CX), DX
	XORQ         BX, BX
	MOVQ         (CX)(BX*1), SI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), DI
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), R8
	ADDQ         AX, BX
	MOVQ         (CX)(BX*1), AX
	MOVQ         $0x0000000f, CX
	MOVQ         CX, X0
	VPBROADCASTB X0, Y0

loop:
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y1
	VMOVDQU (R8), Y4
	VMOVDQU 32(R8), Y5
	VMOVDQU (DI), Y2
	VMOVDQU 32(DI), Y3
	VMOVDQU (AX), Y6
	VMOVDQU 32(AX), Y7
	VPXOR   Y0, Y4, Y4
	VPXOR   Y2, Y6, Y6
	VPXOR   Y1, Y5, Y5
	VPXOR   Y3, Y7, Y7
	VPXOR   Y2, Y0, Y2
	VPXOR   Y3, Y1, Y3
	VPXOR   Y4, Y6, Y6
	VPXOR   Y5, Y7, Y7
	VMOVDQU Y0, (SI)
	VMOVDQU Y1, 32(SI)
	ADDQ    $0x40, SI
	VMOVDQU Y2, (DI)
	VMOVDQU Y3, 32(DI)
	ADDQ    $0x40, DI
	VMOVDQU Y4, (R8)
	VMOVDQU Y5, 32(R8)
	ADDQ    $0x40, R8
	VMOVDQU Y6, (AX)
	VMOVDQU Y7, 32(AX)
	ADDQ    $0x40, AX
	SUBQ    $0x40, DX
	JA      loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_0(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·ifftDIT48_gfni_0(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	VBROADCASTF32X2 t23+40(FP), Z1
	VBROADCASTF32X2 t02+48(FP), Z2
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z3
	VMOVDQU64 (DI), Z4
	VMOVDQU64 (R8), Z5
	VMOVDQU64 (AX), Z6
	VXORPD    Z4, Z3, Z4

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z7
	VXORPD         Z3, Z7, Z3
	VXORPD         Z5, Z6, Z6

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
	VPTERNLOGD     $0x96, Z7, Z3, Z5
	VXORPD         Z4, Z6, Z6

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z2, Z5, Z7
	VXORPD         Z3, Z7, Z3

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
	VXORPD         Z4, Z7, Z4
	VMOVDQU64      Z3, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z4, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z5, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z6, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_0(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·fftDIT48_gfni_0(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	VBROADCASTF32X2 t23+40(FP), Z1
	VBROADCASTF32X2 t02+48(FP), Z2
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z3
	VMOVDQU64 (DI), Z4
	VMOVDQU64 (R8), Z5
	VMOVDQU64 (AX), Z6

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z2, Z5, Z7
	VXORPD         Z3, Z7, Z3

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
	VXORPD         Z4, Z7, Z4
	VXORPD         Z3, Z5, Z5
	VXORPD         Z4, Z6, Z6

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z7
	VXORPD         Z3, Z7, Z3
	VXORPD         Z4, Z3, Z4

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
	VXORPD         Z5, Z7, Z5
	VXORPD         Z5, Z6, Z6
	VMOVDQU64      Z3, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z4, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z5, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z6, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_1(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·ifftDIT48_gfni_1(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t23+40(FP), Z0
	VBROADCASTF32X2 t02+48(FP), Z1
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z2
	VMOVDQU64 (DI), Z3
	VMOVDQU64 (R8), Z4
	VMOVDQU64 (AX), Z5
	VXORPD    Z3, Z2, Z3
	VXORPD    Z4, Z5, Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z5, Z6
	VPTERNLOGD     $0x96, Z6, Z2, Z4
	VXORPD         Z3, Z5, Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
	VXORPD         Z2, Z6, Z2

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
	VXORPD         Z3, Z6, Z3
	VMOVDQU64      Z2, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z3, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z4, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z5, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_1(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·fftDIT48_gfni_1(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	VBROADCASTF32X2 t23+40(FP), Z1
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z2
	VMOVDQU64 (DI), Z3
	VMOVDQU64 (R8), Z4
	VMOVDQU64 (AX), Z5
	VXORPD    Z2, Z4, Z4
	VXORPD    Z3, Z5, Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
	VXORPD         Z2, Z6, Z2
	VXORPD         Z3, Z2, Z3

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
	VXORPD         Z4, Z6, Z4
	VXORPD         Z4, Z5, Z5
	VMOVDQU64      Z2, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z3, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z4, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z5, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_2(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·ifftDIT48_gfni_2(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	VBROADCASTF32X2 t02+48(FP), Z1
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z2
	VMOVDQU64 (DI), Z3
	VMOVDQU64 (R8), Z4
	VMOVDQU64 (AX), Z5
	VXORPD    Z3, Z2, Z3

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
	VXORPD         Z2, Z6, Z2
	VXORPD         Z4, Z5, Z5
	VXORPD         Z2, Z4, Z4
	VXORPD         Z3, Z5, Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
	VXORPD         Z2, Z6, Z2

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
	VXORPD         Z3, Z6, Z3
	VMOVDQU64      Z2, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z3, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z4, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z5, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_2(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·fftDIT48_gfni_2(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t23+40(FP), Z0
	VBROADCASTF32X2 t02+48(FP), Z1
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z2
	VMOVDQU64 (DI), Z3
	VMOVDQU64 (R8), Z4
	VMOVDQU64 (AX), Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
	VXORPD         Z2, Z6, Z2

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
	VXORPD         Z3, Z6, Z3
	VXORPD         Z2, Z4, Z4
	VXORPD         Z3, Z5, Z5
	VXORPD         Z3, Z2, Z3

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z5, Z6
	VXORPD         Z4, Z6, Z4
	VXORPD         Z4, Z5, Z5
	VMOVDQU64      Z2, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z3, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z4, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z5, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_3(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·ifftDIT48_gfni_3(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t02+48(FP), Z0
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z1
	VMOVDQU64 (DI), Z2
	VMOVDQU64 (R8), Z3
	VMOVDQU64 (AX), Z4
	VXORPD    Z2, Z1, Z2
	VXORPD    Z3, Z4, Z4
	VXORPD    Z1, Z3, Z3
	VXORPD    Z2, Z4, Z4

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z5
	VXORPD         Z1, Z5, Z1

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
	VXORPD         Z2, Z5, Z2
	VMOVDQU64      Z1, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z2, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z3, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z4, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_3(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·fftDIT48_gfni_3(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t23+40(FP), Z0
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z1
	VMOVDQU64 (DI), Z2
	VMOVDQU64 (R8), Z3
	VMOVDQU64 (AX), Z4
	VXORPD    Z1, Z3, Z3
	VXORPD    Z2, Z4, Z4
	VXORPD    Z2, Z1, Z2

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
	VXORPD         Z3, Z5, Z3
	VXORPD         Z3, Z4, Z4
	VMOVDQU64      Z1, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z2, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z3, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z4, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_4(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·ifftDIT48_gfni_4(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	VBROADCASTF32X2 t23+40(FP), Z1
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z2
	VMOVDQU64 (DI), Z3
	VMOVDQU64 (R8), Z4
	VMOVDQU64 (AX), Z5
	VXORPD    Z3, Z2, Z3

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
	VXORPD         Z2, Z6, Z2
	VXORPD         Z4, Z5, Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
	VPTERNLOGD     $0x96, Z6, Z2, Z4
	VXORPD         Z3, Z5, Z5
	VMOVDQU64      Z2, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z3, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z4, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z5, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_4(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·fftDIT48_gfni_4(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	VBROADCASTF32X2 t02+48(FP), Z1
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z2
	VMOVDQU64 (DI), Z3
	VMOVDQU64 (R8), Z4
	VMOVDQU64 (AX), Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
	VXORPD         Z2, Z6, Z2

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
	VXORPD         Z3, Z6, Z3
	VXORPD         Z2, Z4, Z4
	VXORPD         Z3, Z5, Z5

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
	VXORPD         Z2, Z6, Z2
	VXORPD         Z3, Z2, Z3
	VXORPD         Z4, Z5, Z5
	VMOVDQU64      Z2, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z3, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z4, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z5, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_5(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·ifftDIT48_gfni_5(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t23+40(FP), Z0
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z1
	VMOVDQU64 (DI), Z2
	VMOVDQU64 (R8), Z3
	VMOVDQU64 (AX), Z4
	VXORPD    Z2, Z1, Z2
	VXORPD    Z3, Z4, Z4

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
	VPTERNLOGD     $0x96, Z5, Z1, Z3
	VXORPD         Z2, Z4, Z4
	VMOVDQU64      Z1, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z2, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z3, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z4, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_5(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·fftDIT48_gfni_5(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z1
	VMOVDQU64 (DI), Z2
	VMOVDQU64 (R8), Z3
	VMOVDQU64 (AX), Z4
	VXORPD    Z1, Z3, Z3
	VXORPD    Z2, Z4, Z4

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z2, Z5
	VXORPD         Z1, Z5, Z1
	VXORPD         Z2, Z1, Z2
	VXORPD         Z3, Z4, Z4
	VMOVDQU64      Z1, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z2, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z3, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z4, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_6(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·ifftDIT48_gfni_6(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t01+32(FP), Z0
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z1
	VMOVDQU64 (DI), Z2
	VMOVDQU64 (R8), Z3
	VMOVDQU64 (AX), Z4
	VXORPD    Z2, Z1, Z2

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z2, Z5
	VXORPD         Z1, Z5, Z1
	VXORPD         Z3, Z4, Z4
	VXORPD         Z1, Z3, Z3
	VXORPD         Z2, Z4, Z4
	VMOVDQU64      Z1, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z2, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z3, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z4, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_6(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F, GFNI
TEXT ·fftDIT48_gfni_6(SB), NOSPLIT, $0-56
	VBROADCASTF32X2 t02+48(FP), Z0
	MOVQ            dist+24(FP), AX
	MOVQ            work_base+0(FP), CX
	MOVQ            8(CX), DX
	XORQ            BX, BX
	MOVQ            (CX)(BX*1), SI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), DI
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), R8
	ADDQ            AX, BX
	MOVQ            (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z1
	VMOVDQU64 (DI), Z2
	VMOVDQU64 (R8), Z3
	VMOVDQU64 (AX), Z4

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z3, Z5
	VXORPD         Z1, Z5, Z1

	// LEO_MULADD_512
	VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
	VXORPD         Z2, Z5, Z2
	VXORPD         Z1, Z3, Z3
	VXORPD         Z2, Z4, Z4
	VXORPD         Z2, Z1, Z2
	VXORPD         Z3, Z4, Z4
	VMOVDQU64      Z1, (SI)
	ADDQ           $0x40, SI
	VMOVDQU64      Z2, (DI)
	ADDQ           $0x40, DI
	VMOVDQU64      Z3, (R8)
	ADDQ           $0x40, R8
	VMOVDQU64      Z4, (AX)
	ADDQ           $0x40, AX
	SUBQ           $0x40, DX
	JA             loop
	VZEROUPPER
	RET

// func ifftDIT48_gfni_7(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F
TEXT ·ifftDIT48_gfni_7(SB), NOSPLIT, $0-56
	MOVQ dist+24(FP), AX
	MOVQ work_base+0(FP), CX
	MOVQ 8(CX), DX
	XORQ BX, BX
	MOVQ (CX)(BX*1), SI
	ADDQ AX, BX
	MOVQ (CX)(BX*1), DI
	ADDQ AX, BX
	MOVQ (CX)(BX*1), R8
	ADDQ AX, BX
	MOVQ (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z0
	VMOVDQU64 (DI), Z1
	VMOVDQU64 (R8), Z2
	VMOVDQU64 (AX), Z3
	VXORPD    Z1, Z0, Z1
	VXORPD    Z2, Z3, Z3
	VXORPD    Z0, Z2, Z2
	VXORPD    Z1, Z3, Z3
	VMOVDQU64 Z0, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z1, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z2, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z3, (AX)
	ADDQ      $0x40, AX
	SUBQ      $0x40, DX
	JA        loop
	VZEROUPPER
	RET

// func fftDIT48_gfni_7(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
// Requires: AVX, AVX512DQ, AVX512F
TEXT ·fftDIT48_gfni_7(SB), NOSPLIT, $0-56
	MOVQ dist+24(FP), AX
	MOVQ work_base+0(FP), CX
	MOVQ 8(CX), DX
	XORQ BX, BX
	MOVQ (CX)(BX*1), SI
	ADDQ AX, BX
	MOVQ (CX)(BX*1), DI
	ADDQ AX, BX
	MOVQ (CX)(BX*1), R8
	ADDQ AX, BX
	MOVQ (CX)(BX*1), AX

loop:
	VMOVDQU64 (SI), Z0
	VMOVDQU64 (DI), Z1
	VMOVDQU64 (R8), Z2
	VMOVDQU64 (AX), Z3
	VXORPD    Z0, Z2, Z2
	VXORPD    Z1, Z3, Z3
	VXORPD    Z1, Z0, Z1
	VXORPD    Z2, Z3, Z3
	VMOVDQU64 Z0, (SI)
	ADDQ      $0x40, SI
	VMOVDQU64 Z1, (DI)
	ADDQ      $0x40, DI
	VMOVDQU64 Z2, (R8)
	ADDQ      $0x40, R8
	VMOVDQU64 Z3, (AX)
	ADDQ      $0x40, AX
	SUBQ      $0x40, DX
	JA        loop
	VZEROUPPER
	RET
