// Code generated by plan9-xlate.pl from CryptoGAMS Perl. DO NOT EDIT.
//
// Run `go generate ./tsasm/arm/...` (or tsasm/arm/regen/regen.sh) to
// regenerate. The upstream chacha-armv4.pl / poly1305-armv4.pl are
// fetched at a pinned commit by regen.sh; see that script for the
// upstream URL and SHA. Translation is by tsasm/arm/regen/plan9-xlate.pl.
//
// The original CryptoGAMS code this was derived from is dual-licensed
// under the BSD-3-Clause and OpenSSL licenses, at the holder's option.
// The CryptoGAMS BSD-3-Clause text is reproduced below; that license
// requires the copyright notice to travel with the source. Files in
// tsasm/arm/regen/ that are not derived from CryptoGAMS (plan9-xlate.pl
// and friends) are BSD-3-Clause without the dual-license option.
//
// ----------------------------------------------------------------
// Copyright (c) 2006, CRYPTOGAMS by <appro@openssl.org>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//       * Redistributions of source code must retain copyright
//         notices, this list of conditions and the following
//         disclaimer.
//
//       * Redistributions in binary form must reproduce the above
//         copyright notice, this list of conditions and the following
//         disclaimer in the documentation and/or other materials
//         provided with the distribution.
//
//       * Neither the name of the CRYPTOGAMS nor the names of its
//         copyright holder and contributors may be used to endorse or
//         promote products derived from this software without specific
//         prior written permission.
//
// ALTERNATIVELY, provided that this notice is retained in full, this
// product may be distributed under the terms of the GNU General Public
// License (GPL), in which case the provisions of the GPL apply INSTEAD
// OF those given above.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// ----------------------------------------------------------------

#include "textflag.h"
#include "funcdata.h"


// func poly1305_init(ctx uintptr, key uintptr)
TEXT ·poly1305_init(SB), NOSPLIT, $36-8
	PCDATA	$PCDATA_UnsafePoint, $-2
	NO_LOCAL_POINTERS
	MOVW	ctx+0(FP), R0
	MOVW	key+4(FP), R1
	MOVW	R4, 4(R13)
	MOVW	R5, 8(R13)
	MOVW	R6, 12(R13)
	MOVW	R7, 16(R13)
	MOVW	R8, 20(R13)
	MOVW	R9, 24(R13)
	MOVW	R11, 32(R13)
lb_Lpoly1305_init:
	EOR	R3, R3, R3
	CMP	$0, R1
	MOVW	R3, 0(R0)	// zero hash value
	MOVW	R3, 4(R0)
	MOVW	R3, 8(R0)
	MOVW	R3, 12(R0)
	MOVW	R3, 16(R0)
	MOVW	R3, 36(R0)	// clear is_base2_26
	ADD	$20, R0, R0
	MOVW.EQ	$0, R0
	B.EQ	lb_Lno_key
	MOVBU	0(R1), R4
	MOVW	R14, 36(R13)
	MOVW	$0x0fffffff, R14
	MOVW	R14, 28(R13)
	MOVBU	1(R1), R5
	BIC	$3, R14, R3	// 0x0ffffffc
	MOVBU	2(R1), R6
	MOVBU	3(R1), R7
	ORR	R5<<8, R4, R4
	MOVBU	4(R1), R5
	ORR	R6<<16, R4, R4
	MOVBU	5(R1), R6
	ORR	R7<<24, R4, R4
	MOVBU	6(R1), R7
	AND	R14, R4, R4
	MOVBU	7(R1), R8
	ORR	R6<<8, R5, R5
	MOVBU	8(R1), R6
	ORR	R7<<16, R5, R5
	MOVBU	9(R1), R7
	ORR	R8<<24, R5, R5
	MOVBU	10(R1), R8
	AND	R3, R5, R5
	MOVBU	11(R1), R9
	ORR	R7<<8, R6, R6
	MOVBU	12(R1), R7
	ORR	R8<<16, R6, R6
	MOVBU	13(R1), R8
	ORR	R9<<24, R6, R6
	MOVBU	14(R1), R9
	AND	R3, R6, R6
	MOVBU	15(R1), R14
	MOVW	R14, 28(R13)
	ORR	R8<<8, R7, R7
	MOVW	R4, 0(R0)
	ORR	R9<<16, R7, R7
	MOVW	R5, 4(R0)
	ORR	R14<<24, R7, R7
	MOVW	R6, 8(R0)
	AND	R3, R7, R7
	MOVW	R7, 12(R0)
	MOVW	$0, R0
	MOVW	36(R13), R14
lb_Lno_key:
	RET

// func poly1305_blocks(ctx uintptr, inp uintptr, len uintptr, padbit uintptr)
TEXT ·poly1305_blocks(SB), NOSPLIT, $72-16
	PCDATA	$PCDATA_UnsafePoint, $-2
	NO_LOCAL_POINTERS
	MOVW	ctx+0(FP), R0
	MOVW	inp+4(FP), R1
	MOVW	len+8(FP), R2
	MOVW	padbit+12(FP), R3
	MOVW	R3, 36(R13)
	MOVW	R4, 40(R13)
	MOVW	R5, 44(R13)
	MOVW	R6, 48(R13)
	MOVW	R7, 52(R13)
	MOVW	R8, 56(R13)
	MOVW	R9, 60(R13)
	MOVW	R11, 68(R13)
lb_Lpoly1305_blocks:
	BIC.S	$15, R2, R2
	B.EQ	lb_Lno_data
	ADD	R1, R2, R2	// end pointer
	MOVW	R14, 72(R13)
	MOVW	0(R0), R4
	MOVW	4(R0), R5
	MOVW	8(R0), R6
	MOVW	12(R0), R7
	MOVW	16(R0), R8
	MOVW	20(R0), R9
	MOVW	24(R0), R14
	MOVW	R14, 64(R13)
	MOVW	28(R0), R11
	MOVW	32(R0), R12
	MOVW	72(R13), R14
	ADD	$20, R0, R0
	MOVW	R2, 20(R13)	// offload stuff
	MOVW	R0, 16(R13)
	MOVW	R1, R14
	CMP	$0, R3
	MOVW	R14, 72(R13)
	MOVW	64(R13), R14
	MOVW	R14, 24(R13)
	MOVW	R11, 28(R13)
	MOVW	R12, 32(R13)
	MOVW	72(R13), R14
	B	lb_Loop
lb_Loop:
	MOVBU.P	16(R14), R0	// load input
	ADD.HI	$1, R8, R8	// 1<<128
	MOVBU	-15(R14), R1
	MOVBU	-14(R14), R2
	MOVBU	-13(R14), R3
	ORR	R1<<8, R0, R1
	MOVBU	-12(R14), R0
	ORR	R2<<16, R1, R2
	MOVBU	-11(R14), R1
	ORR	R3<<24, R2, R3
	MOVBU	-10(R14), R2
	ADD.S	R3, R4, R4	// accumulate input
	MOVBU	-9(R14), R3
	ORR	R1<<8, R0, R1
	MOVBU	-8(R14), R0
	ORR	R2<<16, R1, R2
	MOVBU	-7(R14), R1
	ORR	R3<<24, R2, R3
	MOVBU	-6(R14), R2
	ADC.S	R3, R5, R5
	MOVBU	-5(R14), R3
	ORR	R1<<8, R0, R1
	MOVBU	-4(R14), R0
	ORR	R2<<16, R1, R2
	MOVBU	-3(R14), R1
	ORR	R3<<24, R2, R3
	MOVBU	-2(R14), R2
	ADC.S	R3, R6, R6
	MOVBU	-1(R14), R3
	ORR	R1<<8, R0, R1
	MOVW	R14, 12(R13)	// offload input pointer
	ORR	R2<<16, R1, R2
	MOVW	R14, 72(R13)
	MOVW	64(R13), R14
	ADD	R14>>2, R14, R14
	MOVW	R14, 64(R13)
	ORR	R3<<24, R2, R3
	ADD	R11>>2, R11, R11
	ADC.S	R3, R7, R7
	ADD	R12>>2, R12, R12
	MULLU	R5, R9, (R3, R2)
	ADC	$0, R8, R8
	MULLU	R4, R9, (R1, R0)
	MULALU	R8, R14, (R3, R2)
	MULALU	R7, R14, (R1, R0)
	MOVW	24(R13), R14	// reload r10
	MOVW	R14, 64(R13)
	MULALU	R6, R12, (R3, R2)
	MULALU	R5, R12, (R1, R0)
	MULALU	R7, R11, (R3, R2)
	MULALU	R6, R11, (R1, R0)
	MULALU	R4, R14, (R3, R2)
	MOVW	R0, 4(R13)	// future r4
	MUL	R8, R11, R0
	MOVW	28(R13), R11	// reload r11
	ADD.S	R1, R2, R2	// d1+=d0>>32
	EOR	R1, R1, R1
	MOVW	72(R13), R14
	ADC	$0, R3, R14	// future r6
	MOVW	R2, 8(R13)	// future r5
	MUL	R8, R12, R2
	EOR	R3, R3, R3
	MULALU	R7, R12, (R1, R0)
	MOVW	32(R13), R12	// reload r12
	MULALU	R7, R9, (R3, R2)
	MULALU	R6, R9, (R1, R0)
	MOVW	R14, 72(R13)
	MOVW	64(R13), R14
	MULALU	R6, R14, (R3, R2)
	MULALU	R5, R14, (R1, R0)
	MULALU	R5, R11, (R3, R2)
	MULALU	R4, R11, (R1, R0)
	MULALU	R4, R12, (R3, R2)
	MOVW	4(R13), R4
	MUL	R8, R9, R8
	MOVW	8(R13), R5
	MOVW	72(R13), R14
	ADD.S	R0, R14, R6	// d2+=d1>>32
	MOVW	12(R13), R14	// reload input pointer
	ADC	$0, R1, R1
	ADD.S	R1, R2, R7	// d3+=d2>>32
	MOVW	20(R13), R0	// reload end pointer
	ADC	$0, R3, R3
	ADD	R3, R8, R8	// h4+=d3>>32
	BIC	$3, R8, R1
	AND	$3, R8, R8
	ADD	R1>>2, R1, R1	// *=5
	ADD.S	R1, R4, R4
	ADC.S	$0, R5, R5
	ADC.S	$0, R6, R6
	ADC.S	$0, R7, R7
	ADC	$0, R8, R8
	CMP	R14, R0	// done yet?
	B.HI	lb_Loop
	MOVW	16(R13), R0
	MOVM.DB	[R4, R5, R6, R7, R8], (R0)	// store the result
lb_Lno_data:
	RET

// func poly1305_emit(ctx uintptr, mac uintptr, nonce uintptr)
TEXT ·poly1305_emit(SB), NOSPLIT, $36-12
	PCDATA	$PCDATA_UnsafePoint, $-2
	NO_LOCAL_POINTERS
	MOVW	ctx+0(FP), R0
	MOVW	mac+4(FP), R1
	MOVW	nonce+8(FP), R2
	MOVW	R4, 4(R13)
	MOVW	R5, 8(R13)
	MOVW	R6, 12(R13)
	MOVW	R7, 16(R13)
	MOVW	R8, 20(R13)
	MOVW	R9, 24(R13)
	MOVW	R11, 32(R13)
lb_Lpoly1305_emit:
	MOVM.IA	(R0), [R3, R4, R5, R6, R7]
	ADD.S	$5, R3, R8	// compare to modulus
	ADC.S	$0, R4, R9
	MOVW	R14, 36(R13)
	ADC.S	$0, R5, R14
	MOVW	R14, 28(R13)
	ADC.S	$0, R6, R11
	ADC	$0, R7, R0
	TST	$4, R0	// did it carry/borrow?
	MOVW.NE	R8, R3
	MOVW	0(R2), R8
	MOVW.NE	R9, R4
	MOVW	4(R2), R9
	MOVW.NE	R14, R5
	MOVW	8(R2), R14
	MOVW	R14, 28(R13)
	MOVW.NE	R11, R6
	MOVW	12(R2), R11
	ADD.S	R8, R3, R3
	ADC.S	R9, R4, R4
	ADC.S	R14, R5, R5
	ADC	R11, R6, R6
	MOVB	R3, 0(R1)
	MOVW	R3>>8, R3
	MOVB	R4, 4(R1)
	MOVW	R4>>8, R4
	MOVB	R5, 8(R1)
	MOVW	R5>>8, R5
	MOVB	R6, 12(R1)
	MOVW	R6>>8, R6
	MOVB	R3, 1(R1)
	MOVW	R3>>8, R3
	MOVB	R4, 5(R1)
	MOVW	R4>>8, R4
	MOVB	R5, 9(R1)
	MOVW	R5>>8, R5
	MOVB	R6, 13(R1)
	MOVW	R6>>8, R6
	MOVB	R3, 2(R1)
	MOVW	R3>>8, R3
	MOVB	R4, 6(R1)
	MOVW	R4>>8, R4
	MOVB	R5, 10(R1)
	MOVW	R5>>8, R5
	MOVB	R6, 14(R1)
	MOVW	R6>>8, R6
	MOVB	R3, 3(R1)
	MOVB	R4, 7(R1)
	MOVB	R5, 11(R1)
	MOVB	R6, 15(R1)
	RET
TEXT ·poly1305_init_neon(SB), NOSPLIT, $0-8
	PCDATA	$PCDATA_UnsafePoint, $-2
	NO_LOCAL_POINTERS
	MOVW	ctx+0(FP), R0
	MOVW	key+4(FP), R1
lb_Lpoly1305_init_neon:
	MOVW	48(R0), R3	// first table element
	CMP	$-1, R3	// is value impossible?
	B.NE	lb_Lno_init_neon
	MOVW	20(R0), R4	// load key base 2^32
	MOVW	24(R0), R5
	MOVW	28(R0), R6
	MOVW	32(R0), R7
	AND	$0x03ffffff, R4, R2	// base 2^32 -> base 2^26
	MOVW	R4>>26, R3
	MOVW	R5>>20, R4
	ORR	R5<<6, R3, R3
	MOVW	R6>>14, R5
	ORR	R6<<12, R4, R4
	MOVW	R7>>8, R6
	ORR	R7<<18, R5, R5
	AND	$0x03ffffff, R3, R3
	AND	$0x03ffffff, R4, R4
	AND	$0x03ffffff, R5, R5
	WORD	$0xee802b10	// vdup.32 d0,r2
	ADD	R3<<2, R3, R2	// *5
	WORD	$0xee813b10	// vdup.32 d1,r3
	ADD	R4<<2, R4, R3
	WORD	$0xee822b10	// vdup.32 d2,r2
	WORD	$0xee834b10	// vdup.32 d3,r4
	ADD	R5<<2, R5, R4
	WORD	$0xee843b10	// vdup.32 d4,r3
	WORD	$0xee855b10	// vdup.32 d5,r5
	ADD	R6<<2, R6, R5
	WORD	$0xee864b10	// vdup.32 d6,r4
	WORD	$0xee876b10	// vdup.32 d7,r6
	WORD	$0xee885b10	// vdup.32 d8,r5
	MOVW	$2, R5	// counter
lb_Lsquare_neon:
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
	// d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
	// d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
	// d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
	// d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
	WORD	$0xf3a0aa60	// vmull.u32 q5,d0,d0[1]
	WORD	$0xf3a1ca60	// vmull.u32 q6,d1,d0[1]
	WORD	$0xf3a3ea60	// vmull.u32 q7,d3,d0[1]
	WORD	$0xf3e50a60	// vmull.u32 q8,d5,d0[1]
	WORD	$0xf3e72a60	// vmull.u32 q9,d7,d0[1]
	WORD	$0xf3a7a262	// vmlal.u32 q5,d7,d2[1]
	WORD	$0xf3a0c261	// vmlal.u32 q6,d0,d1[1]
	WORD	$0xf3a1e261	// vmlal.u32 q7,d1,d1[1]
	WORD	$0xf3e30261	// vmlal.u32 q8,d3,d1[1]
	WORD	$0xf3e52261	// vmlal.u32 q9,d5,d1[1]
	WORD	$0xf3a5a264	// vmlal.u32 q5,d5,d4[1]
	WORD	$0xf3a7c264	// vmlal.u32 q6,d7,d4[1]
	WORD	$0xf3e10263	// vmlal.u32 q8,d1,d3[1]
	WORD	$0xf3a0e263	// vmlal.u32 q7,d0,d3[1]
	WORD	$0xf3e32263	// vmlal.u32 q9,d3,d3[1]
	WORD	$0xf3a3a266	// vmlal.u32 q5,d3,d6[1]
	WORD	$0xf3e00265	// vmlal.u32 q8,d0,d5[1]
	WORD	$0xf3a5c266	// vmlal.u32 q6,d5,d6[1]
	WORD	$0xf3a7e266	// vmlal.u32 q7,d7,d6[1]
	WORD	$0xf3e12265	// vmlal.u32 q9,d1,d5[1]
	WORD	$0xf3e70268	// vmlal.u32 q8,d7,d8[1]
	WORD	$0xf3a1a268	// vmlal.u32 q5,d1,d8[1]
	WORD	$0xf3a3c268	// vmlal.u32 q6,d3,d8[1]
	WORD	$0xf3a5e268	// vmlal.u32 q7,d5,d8[1]
	WORD	$0xf3e02267	// vmlal.u32 q9,d0,d7[1]
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
	// and P. Schwabe
	// H0>>+H1>>+H2>>+H3>>+H4
	// H3>>+H4>>*5+H0>>+H1
	// Trivia.
	// Result of multiplication of n-bit number by m-bit number is
	// n+m bits wide. However! Even though 2^n is a n+1-bit number,
	// m-bit number multiplied by 2^n is still n+m bits wide.
	// Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
	// and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
	// one is n+1 bits wide.
	// >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
	// H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
	// can be 27. However! In cases when their width exceeds 26 bits
	// they are limited by 2^26+2^6. This in turn means that *sum*
	// of the products with these values can still be viewed as sum
	// of 52-bit numbers as long as the amount of addends is not a
	// power of 2. For example,
	// H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
	// which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
	// 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
	// 8 * (2^52) or 2^55. However, the value is then multiplied by
	// by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
	// which is less than 32 * (2^52) or 2^57. And when processing
	// data we are looking at triple as many addends...
	// In key setup procedure pre-reduced H0 is limited by 5*4+1 and
	// 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
	// input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
	// 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
	// instruction accepts 2x32-bit input and writes 2x64-bit result.
	// This means that result of reduction have to be compressed upon
	// loop wrap-around. This can be done in the process of reduction
	// to minimize amount of instructions [as well as amount of
	// 128-bit instructions, which benefits low-end processors], but
	// one has to watch for H2 (which is narrower than H0) and 5*H4
	// not being wider than 58 bits, so that result of right shift
	// by 26 bits fits in 32 bits. This is also useful on x86,
	// because it allows to use paddd in place for paddq, which
	// benefits Atom, where paddq is ridiculously slow.
	WORD	$0xf3e6e0f0	// vshr.u64 q15,q8,#26
	WORD	$0xf3fa0220	// vmovn.i64 d16,q8
	WORD	$0xf3a680da	// vshr.u64 q4,q5,#26
	WORD	$0xf3baa20a	// vmovn.i64 d10,q5
	WORD	$0xf27228ee	// vadd.i64 q9,q9,q15
	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
	WORD	$0xf23cc848	// vadd.i64 q6,q6,q4
	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
	WORD	$0xf2e6e832	// vshrn.u64 d30,q9,#26
	WORD	$0xf3fa2222	// vmovn.i64 d18,q9
	WORD	$0xf3a680dc	// vshr.u64 q4,q6,#26
	WORD	$0xf3bac20c	// vmovn.i64 d12,q6
	WORD	$0xf23ee848	// vadd.i64 q7,q7,q4
	WORD	$0xf3c7273c	// vbic.i32 d18,#0xfc000000
	WORD	$0xf387c73c	// vbic.i32 d12,#0xfc000000
	WORD	$0xf22aa82e	// vadd.i32 d10,d10,d30
	WORD	$0xf2e2e53e	// vshl.u32 d30,d30,#2
	WORD	$0xf2a6881e	// vshrn.u64 d8,q7,#26
	WORD	$0xf3bae20e	// vmovn.i64 d14,q7
	WORD	$0xf22aa82e	// vadd.i32 d10,d10,d30
	WORD	$0xf2600888	// vadd.i32 d16,d16,d8
	WORD	$0xf387e73c	// vbic.i32 d14,#0xfc000000
	WORD	$0xf3e6e01a	// vshr.u32 d30,d10,#26
	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
	WORD	$0xf3a68030	// vshr.u32 d8,d16,#26
	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
	WORD	$0xf22cc82e	// vadd.i32 d12,d12,d30
	WORD	$0xf2622888	// vadd.i32 d18,d18,d8
	SUB.S	$1, R5, R5
	B.EQ	lb_Lsquare_break_neon
	ADD	$(48+0*9*4), R0, R6
	ADD	$(48+1*9*4), R0, R7
	WORD	$0xf3ba008a	// vtrn.32 d0,d10
	WORD	$0xf3ba308e	// vtrn.32 d3,d14
	WORD	$0xf3ba50a0	// vtrn.32 d5,d16
	WORD	$0xf3ba108c	// vtrn.32 d1,d12
	WORD	$0xf3ba70a2	// vtrn.32 d7,d18
	WORD	$0xf2a24513	// vshl.u32 d4,d3,#2
	WORD	$0xf2a26515	// vshl.u32 d6,d5,#2
	WORD	$0xf2a22511	// vshl.u32 d2,d1,#2
	WORD	$0xf2a28517	// vshl.u32 d8,d7,#2
	WORD	$0xf2244803	// vadd.i32 d4,d4,d3
	WORD	$0xf2222801	// vadd.i32 d2,d2,d1
	WORD	$0xf2266805	// vadd.i32 d6,d6,d5
	WORD	$0xf2288807	// vadd.i32 d8,d8,d7
	WORD	$0xf4860b0d	// vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
	WORD	$0xf4870b8d	// vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
	WORD	$0xf4864b0d	// vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
	WORD	$0xf4874b8d	// vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
	WORD	$0xf486883f	// vst1.32 {d8[0]},[r6,:32]
	WORD	$0xf48788bf	// vst1.32 {d8[1]},[r7,:32]
	B	lb_Lsquare_neon
lb_Lsquare_break_neon:
	ADD	$(48+2*4*9), R0, R6
	ADD	$(48+3*4*9), R0, R7
	WORD	$0xf22a011a	// vmov d0,d10
	WORD	$0xf2a2251c	// vshl.u32 d2,d12,#2
	WORD	$0xf22c111c	// vmov d1,d12
	WORD	$0xf2a2451e	// vshl.u32 d4,d14,#2
	WORD	$0xf22e311e	// vmov d3,d14
	WORD	$0xf2a26530	// vshl.u32 d6,d16,#2
	WORD	$0xf22051b0	// vmov d5,d16
	WORD	$0xf2a28532	// vshl.u32 d8,d18,#2
	WORD	$0xf22271b2	// vmov d7,d18
	WORD	$0xf222280c	// vadd.i32 d2,d2,d12
	WORD	$0xf224480e	// vadd.i32 d4,d4,d14
	WORD	$0xf2266820	// vadd.i32 d6,d6,d16
	WORD	$0xf2288822	// vadd.i32 d8,d8,d18
	WORD	$0xf4860b0d	// vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
	WORD	$0xf4870b8d	// vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
	WORD	$0xf4864b0d	// vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
	WORD	$0xf4874b8d	// vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
	WORD	$0xf486880f	// vst1.32 {d8[0]},[r6]
	WORD	$0xf487888f	// vst1.32 {d8[1]},[r7]
lb_Lno_init_neon:
	RET

// func poly1305_blocks_neon(ctx uintptr, inp uintptr, len uintptr, padbit uintptr)
TEXT ·poly1305_blocks_neon(SB), NOSPLIT, $96-16
	PCDATA	$PCDATA_UnsafePoint, $-2
	NO_LOCAL_POINTERS
	MOVW	ctx+0(FP), R0
	MOVW	inp+4(FP), R1
	MOVW	len+8(FP), R2
	MOVW	padbit+12(FP), R3
	MOVW	R4, 84(R13)
	MOVW	R5, 88(R13)
	MOVW	R6, 92(R13)
	MOVW	R7, 96(R13)
lb_Lpoly1305_blocks_neon:
	MOVW	36(R0), R12	// is_base2_26
	CMP	$64, R2
	TST	R12, R12	// is_base2_26?
	B.NE	lb_Lbase2_26_neon
	MOVW	R1, 4(R13)
	MOVW	R2, 8(R13)
	MOVW	R3, 12(R13)
	BL	·poly1305_init_neon(SB)
	MOVW	0(R0), R4	// load hash value base 2^32
	MOVW	4(R0), R5
	MOVW	8(R0), R6
	MOVW	12(R0), R7
	MOVW	16(R0), R12
	AND	$0x03ffffff, R4, R2	// base 2^32 -> base 2^26
	MOVW	R4>>26, R3
	WORD	$0xf30aa11a	// veor d10,d10,d10
	MOVW	R5>>20, R4
	ORR	R5<<6, R3, R3
	WORD	$0xf30cc11c	// veor d12,d12,d12
	MOVW	R6>>14, R5
	ORR	R6<<12, R4, R4
	WORD	$0xf30ee11e	// veor d14,d14,d14
	MOVW	R7>>8, R6
	ORR	R7<<18, R5, R5
	WORD	$0xf34001b0	// veor d16,d16,d16
	AND	$0x03ffffff, R3, R3
	ORR	R12<<24, R6, R6
	WORD	$0xf34221b2	// veor d18,d18,d18
	AND	$0x03ffffff, R4, R4
	MOVW	$1, R1
	AND	$0x03ffffff, R5, R5
	MOVW	R1, 36(R0)	// set is_base2_26
	WORD	$0xee0a2b10	// vmov.32 d10[0],r2
	WORD	$0xee0c3b10	// vmov.32 d12[0],r3
	WORD	$0xee0e4b10	// vmov.32 d14[0],r4
	WORD	$0xee005b90	// vmov.32 d16[0],r5
	WORD	$0xee026b90	// vmov.32 d18[0],r6
	MOVW	$zeros<>(SB), R5
	MOVM.IA.W	(R13), [R1, R2, R3, R14]
	B	lb_Lhash_loaded
lb_Lbase2_26_neon:
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// load hash value
	WORD	$0xf30aa11a	// veor d10,d10,d10
	WORD	$0xf30cc11c	// veor d12,d12,d12
	WORD	$0xf30ee11e	// veor d14,d14,d14
	WORD	$0xf34001b0	// veor d16,d16,d16
	WORD	$0xf34221b2	// veor d18,d18,d18
	WORD	$0xf4a0ab4d	// vld4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]!
	MOVW	$zeros<>(SB), R5
	WORD	$0xf4e0280f	// vld1.32 {d18[0]},[r0]
	SUB	$16, R0, R0	// rewind
lb_Lhash_loaded:
	ADD	$32, R1, R4
	MOVW	R3<<24, R3
	TST	$31, R2
	B.EQ	lb_Leven
	WORD	$0xf4e14b4d	// vld4.32 {d20[0],d22[0],d24[0],d26[0]},[r1]!
	WORD	$0xee0c3b90	// vmov.32 d28[0],r3
	SUB	$16, R2, R2
	ADD	$32, R1, R4
	WORD	$0xf3f8c43a	// vsri.u32 d28,d26,#8
	WORD	$0xf2f2a53a	// vshl.u32 d26,d26,#18
	WORD	$0xf3f2a438	// vsri.u32 d26,d24,#14
	WORD	$0xf2ec8538	// vshl.u32 d24,d24,#12
	WORD	$0xf26cd8a2	// vadd.i32 d29,d28,d18
	WORD	$0xf3c7a73c	// vbic.i32 d26,#0xfc000000
	WORD	$0xf3ec8436	// vsri.u32 d24,d22,#20
	WORD	$0xf2e66536	// vshl.u32 d22,d22,#6
	WORD	$0xf3c7873c	// vbic.i32 d24,#0xfc000000
	WORD	$0xf3e66434	// vsri.u32 d22,d20,#26
	WORD	$0xf26ab8a0	// vadd.i32 d27,d26,d16
	WORD	$0xf3c7473c	// vbic.i32 d20,#0xfc000000
	WORD	$0xf3c7673c	// vbic.i32 d22,#0xfc000000
	WORD	$0xf268988e	// vadd.i32 d25,d24,d14
	WORD	$0xf264588a	// vadd.i32 d21,d20,d10
	WORD	$0xf266788c	// vadd.i32 d23,d22,d12
	MOVW	R5, R7
	ADD	$48, R0, R6
	CMP	R2, R2
	B	lb_Long_tail
lb_Leven:
	SUB.S	$64, R2, R2
	MOVW.LO	R5, R4
	WORD	$0xf2c0c651	// vmov.i32 q14,#1<<24
	WORD	$0xf461418f	// vld4.32 {d20,d22,d24,d26},[r1]
	ADD	$64, R1, R1
	WORD	$0xf464518f	// vld4.32 {d21,d23,d25,d27},[r4]
	ADD	$64, R4, R4
	ADD.HI	$(48+1*9*4), R0, R7
	ADD.HI	$(48+3*9*4), R0, R6
	WORD	$0xf3f8c47a	// vsri.u32 q14,q13,#8
	WORD	$0xf2f2a57a	// vshl.u32 q13,q13,#18
	WORD	$0xf3f2a478	// vsri.u32 q13,q12,#14
	WORD	$0xf2ec8578	// vshl.u32 q12,q12,#12
	WORD	$0xf3c7a77c	// vbic.i32 q13,#0xfc000000
	WORD	$0xf3ec8476	// vsri.u32 q12,q11,#20
	WORD	$0xf2e66576	// vshl.u32 q11,q11,#6
	WORD	$0xf3c7877c	// vbic.i32 q12,#0xfc000000
	WORD	$0xf3e66474	// vsri.u32 q11,q10,#26
	WORD	$0xf3c7477c	// vbic.i32 q10,#0xfc000000
	WORD	$0xf3c7677c	// vbic.i32 q11,#0xfc000000
	B.LS	lb_Lskip_loop
	WORD	$0xf4a70b8d	// vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
	WORD	$0xf4a60b0d	// vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
	WORD	$0xf4a74b8d	// vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
	WORD	$0xf4a64b0d	// vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
	B	lb_Loop_neon
lb_Loop_neon:
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
	// ___________________/
	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
	// ___________________/ ____________________/
	// Note that we start with inp[2:3]*r^2. This is because it
	// doesn't depend on reduction in previous iteration.
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
	// d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
	// d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
	// d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
	// d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// inp[2:3]*r^2
	WORD	$0xf268888e	// vadd.i32 d24,d24,d14
	WORD	$0xf3a9eae0	// vmull.u32 q7,d25,d0[1]
	WORD	$0xf264488a	// vadd.i32 d20,d20,d10
	WORD	$0xf3a5aae0	// vmull.u32 q5,d21,d0[1]
	WORD	$0xf26aa8a0	// vadd.i32 d26,d26,d16
	WORD	$0xf3eb0ae0	// vmull.u32 q8,d27,d0[1]
	WORD	$0xf3a7e2e1	// vmlal.u32 q7,d23,d1[1]
	WORD	$0xf266688c	// vadd.i32 d22,d22,d12
	WORD	$0xf3a7cae0	// vmull.u32 q6,d23,d0[1]
	WORD	$0xf26cc8a2	// vadd.i32 d28,d28,d18
	WORD	$0xf3ed2ae0	// vmull.u32 q9,d29,d0[1]
	SUB.S	$64, R2, R2
	WORD	$0xf3ada2e2	// vmlal.u32 q5,d29,d2[1]
	MOVW.LO	R5, R4
	WORD	$0xf3e902e1	// vmlal.u32 q8,d25,d1[1]
	WORD	$0xf4a788bf	// vld1.32 d8[1],[r7,:32]
	WORD	$0xf3a5c2e1	// vmlal.u32 q6,d21,d1[1]
	WORD	$0xf3eb22e1	// vmlal.u32 q9,d27,d1[1]
	WORD	$0xf3aba2e4	// vmlal.u32 q5,d27,d4[1]
	WORD	$0xf3e702e3	// vmlal.u32 q8,d23,d3[1]
	WORD	$0xf3e922e3	// vmlal.u32 q9,d25,d3[1]
	WORD	$0xf3adc2e4	// vmlal.u32 q6,d29,d4[1]
	WORD	$0xf3a5e2e3	// vmlal.u32 q7,d21,d3[1]
	WORD	$0xf3e502e5	// vmlal.u32 q8,d21,d5[1]
	WORD	$0xf3a9a2e6	// vmlal.u32 q5,d25,d6[1]
	WORD	$0xf3e722e5	// vmlal.u32 q9,d23,d5[1]
	WORD	$0xf3abc2e6	// vmlal.u32 q6,d27,d6[1]
	WORD	$0xf3ade2e6	// vmlal.u32 q7,d29,d6[1]
	WORD	$0xf3ed02e8	// vmlal.u32 q8,d29,d8[1]
	WORD	$0xf3a7a2e8	// vmlal.u32 q5,d23,d8[1]
	WORD	$0xf3e522e7	// vmlal.u32 q9,d21,d7[1]
	WORD	$0xf3a9c2e8	// vmlal.u32 q6,d25,d8[1]
	WORD	$0xf3abe2e8	// vmlal.u32 q7,d27,d8[1]
	WORD	$0xf464518f	// vld4.32 {d21,d23,d25,d27},[r4]
	ADD	$64, R4, R4
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// (hash+inp[0:1])*r^4 and accumulate
	WORD	$0xf3ea02c0	// vmlal.u32 q8,d26,d0[0]
	WORD	$0xf3a4a2c0	// vmlal.u32 q5,d20,d0[0]
	WORD	$0xf3ec22c0	// vmlal.u32 q9,d28,d0[0]
	WORD	$0xf3a6c2c0	// vmlal.u32 q6,d22,d0[0]
	WORD	$0xf3a8e2c0	// vmlal.u32 q7,d24,d0[0]
	WORD	$0xf4a6883f	// vld1.32 d8[0],[r6,:32]
	WORD	$0xf3e802c1	// vmlal.u32 q8,d24,d1[0]
	WORD	$0xf3aca2c2	// vmlal.u32 q5,d28,d2[0]
	WORD	$0xf3ea22c1	// vmlal.u32 q9,d26,d1[0]
	WORD	$0xf3a4c2c1	// vmlal.u32 q6,d20,d1[0]
	WORD	$0xf3a6e2c1	// vmlal.u32 q7,d22,d1[0]
	WORD	$0xf3e602c3	// vmlal.u32 q8,d22,d3[0]
	WORD	$0xf3aaa2c4	// vmlal.u32 q5,d26,d4[0]
	WORD	$0xf3e822c3	// vmlal.u32 q9,d24,d3[0]
	WORD	$0xf3acc2c4	// vmlal.u32 q6,d28,d4[0]
	WORD	$0xf3a4e2c3	// vmlal.u32 q7,d20,d3[0]
	WORD	$0xf3e402c5	// vmlal.u32 q8,d20,d5[0]
	WORD	$0xf3a8a2c6	// vmlal.u32 q5,d24,d6[0]
	WORD	$0xf3e622c5	// vmlal.u32 q9,d22,d5[0]
	WORD	$0xf3aac2c6	// vmlal.u32 q6,d26,d6[0]
	WORD	$0xf3ec02c8	// vmlal.u32 q8,d28,d8[0]
	WORD	$0xf3ace2c6	// vmlal.u32 q7,d28,d6[0]
	WORD	$0xf3a6a2c8	// vmlal.u32 q5,d22,d8[0]
	WORD	$0xf3e422c7	// vmlal.u32 q9,d20,d7[0]
	WORD	$0xf2c0c651	// vmov.i32 q14,#1<<24
	WORD	$0xf3a8c2c8	// vmlal.u32 q6,d24,d8[0]
	WORD	$0xf3aae2c8	// vmlal.u32 q7,d26,d8[0]
	WORD	$0xf461418f	// vld4.32 {d20,d22,d24,d26},[r1]
	ADD	$64, R1, R1
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// lazy reduction interleaved with base 2^32 -> base 2^26 of
	// inp[0:3] previously loaded to q10-q13 and smashed to q10-q14.
	WORD	$0xf3e6e0f0	// vshr.u64 q15,q8,#26
	WORD	$0xf3fa0220	// vmovn.i64 d16,q8
	WORD	$0xf3a680da	// vshr.u64 q4,q5,#26
	WORD	$0xf3baa20a	// vmovn.i64 d10,q5
	WORD	$0xf27228ee	// vadd.i64 q9,q9,q15
	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
	WORD	$0xf3f8c47a	// vsri.u32 q14,q13,#8
	WORD	$0xf23cc848	// vadd.i64 q6,q6,q4
	WORD	$0xf2f2a57a	// vshl.u32 q13,q13,#18
	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
	WORD	$0xf2e6e832	// vshrn.u64 d30,q9,#26
	WORD	$0xf3fa2222	// vmovn.i64 d18,q9
	WORD	$0xf3a680dc	// vshr.u64 q4,q6,#26
	WORD	$0xf3bac20c	// vmovn.i64 d12,q6
	WORD	$0xf23ee848	// vadd.i64 q7,q7,q4
	WORD	$0xf3f2a478	// vsri.u32 q13,q12,#14
	WORD	$0xf3c7273c	// vbic.i32 d18,#0xfc000000
	WORD	$0xf2ec8578	// vshl.u32 q12,q12,#12
	WORD	$0xf387c73c	// vbic.i32 d12,#0xfc000000
	WORD	$0xf22aa82e	// vadd.i32 d10,d10,d30
	WORD	$0xf2e2e53e	// vshl.u32 d30,d30,#2
	WORD	$0xf3c7a77c	// vbic.i32 q13,#0xfc000000
	WORD	$0xf2a6881e	// vshrn.u64 d8,q7,#26
	WORD	$0xf3bae20e	// vmovn.i64 d14,q7
	WORD	$0xf3aaa02e	// vaddl.u32 q5,d10,d30
	WORD	$0xf3ec8476	// vsri.u32 q12,q11,#20
	WORD	$0xf2600888	// vadd.i32 d16,d16,d8
	WORD	$0xf2e66576	// vshl.u32 q11,q11,#6
	WORD	$0xf387e73c	// vbic.i32 d14,#0xfc000000
	WORD	$0xf3c7877c	// vbic.i32 q12,#0xfc000000
	WORD	$0xf2e6e81a	// vshrn.u64 d30,q5,#26
	WORD	$0xf3baa20a	// vmovn.i64 d10,q5
	WORD	$0xf3e66474	// vsri.u32 q11,q10,#26
	WORD	$0xf3c7477c	// vbic.i32 q10,#0xfc000000
	WORD	$0xf3a68030	// vshr.u32 d8,d16,#26
	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
	WORD	$0xf22cc82e	// vadd.i32 d12,d12,d30
	WORD	$0xf2622888	// vadd.i32 d18,d18,d8
	WORD	$0xf3c7677c	// vbic.i32 q11,#0xfc000000
	B.HI	lb_Loop_neon
lb_Lskip_loop:
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
	ADD	$(48+0*9*4), R0, R7
	ADD	$(48+1*9*4), R0, R6
	ADD.S	$32, R2, R2
	MOVW.NE	$0, R2
	B.NE	lb_Long_tail
	WORD	$0xf268988e	// vadd.i32 d25,d24,d14
	WORD	$0xf264588a	// vadd.i32 d21,d20,d10
	WORD	$0xf26ab8a0	// vadd.i32 d27,d26,d16
	WORD	$0xf266788c	// vadd.i32 d23,d22,d12
	WORD	$0xf26cd8a2	// vadd.i32 d29,d28,d18
lb_Long_tail:
	WORD	$0xf4a70b8d	// vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
	WORD	$0xf4a60b0d	// vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
	WORD	$0xf268888e	// vadd.i32 d24,d24,d14
	WORD	$0xf3a9ec80	// vmull.u32 q7,d25,d0
	WORD	$0xf264488a	// vadd.i32 d20,d20,d10
	WORD	$0xf3a5ac80	// vmull.u32 q5,d21,d0
	WORD	$0xf26aa8a0	// vadd.i32 d26,d26,d16
	WORD	$0xf3eb0c80	// vmull.u32 q8,d27,d0
	WORD	$0xf266688c	// vadd.i32 d22,d22,d12
	WORD	$0xf3a7cc80	// vmull.u32 q6,d23,d0
	WORD	$0xf26cc8a2	// vadd.i32 d28,d28,d18
	WORD	$0xf3ed2c80	// vmull.u32 q9,d29,d0
	WORD	$0xf3ada882	// vmlal.u32 q5,d29,d2
	WORD	$0xf4a74b8d	// vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
	WORD	$0xf3e90881	// vmlal.u32 q8,d25,d1
	WORD	$0xf4a64b0d	// vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
	WORD	$0xf3a5c881	// vmlal.u32 q6,d21,d1
	WORD	$0xf3eb2881	// vmlal.u32 q9,d27,d1
	WORD	$0xf3a7e881	// vmlal.u32 q7,d23,d1
	WORD	$0xf3e70883	// vmlal.u32 q8,d23,d3
	WORD	$0xf4a788bf	// vld1.32 d8[1],[r7,:32]
	WORD	$0xf3aba884	// vmlal.u32 q5,d27,d4
	WORD	$0xf4a6883f	// vld1.32 d8[0],[r6,:32]
	WORD	$0xf3e92883	// vmlal.u32 q9,d25,d3
	WORD	$0xf3adc884	// vmlal.u32 q6,d29,d4
	WORD	$0xf3a5e883	// vmlal.u32 q7,d21,d3
	WORD	$0xf3e50885	// vmlal.u32 q8,d21,d5
	ADD.NE	$(48+2*9*4), R0, R7
	WORD	$0xf3a9a886	// vmlal.u32 q5,d25,d6
	ADD.NE	$(48+3*9*4), R0, R6
	WORD	$0xf3e72885	// vmlal.u32 q9,d23,d5
	WORD	$0xf3abc886	// vmlal.u32 q6,d27,d6
	WORD	$0xf3ade886	// vmlal.u32 q7,d29,d6
	WORD	$0xf3ed0888	// vmlal.u32 q8,d29,d8
	WORD	$0xf2300150	// vorn q0,q0,q0
	WORD	$0xf3a7a888	// vmlal.u32 q5,d23,d8
	WORD	$0xf39a00d0	// vshr.u64 q0,q0,#38
	WORD	$0xf3e52887	// vmlal.u32 q9,d21,d7
	WORD	$0xf3a9c888	// vmlal.u32 q6,d25,d8
	WORD	$0xf3abe888	// vmlal.u32 q7,d27,d8
	B.EQ	lb_Lshort_tail
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// (hash+inp[0:1])*r^4:r^3 and accumulate
	WORD	$0xf4a70b8d	// vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
	WORD	$0xf4a60b0d	// vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
	WORD	$0xf3a8e880	// vmlal.u32 q7,d24,d0
	WORD	$0xf3a4a880	// vmlal.u32 q5,d20,d0
	WORD	$0xf3ea0880	// vmlal.u32 q8,d26,d0
	WORD	$0xf3a6c880	// vmlal.u32 q6,d22,d0
	WORD	$0xf3ec2880	// vmlal.u32 q9,d28,d0
	WORD	$0xf3aca882	// vmlal.u32 q5,d28,d2
	WORD	$0xf4a74b8d	// vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
	WORD	$0xf3e80881	// vmlal.u32 q8,d24,d1
	WORD	$0xf4a64b0d	// vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
	WORD	$0xf3a4c881	// vmlal.u32 q6,d20,d1
	WORD	$0xf3ea2881	// vmlal.u32 q9,d26,d1
	WORD	$0xf3a6e881	// vmlal.u32 q7,d22,d1
	WORD	$0xf3e60883	// vmlal.u32 q8,d22,d3
	WORD	$0xf4a788bf	// vld1.32 d8[1],[r7,:32]
	WORD	$0xf3aaa884	// vmlal.u32 q5,d26,d4
	WORD	$0xf4a6883f	// vld1.32 d8[0],[r6,:32]
	WORD	$0xf3e82883	// vmlal.u32 q9,d24,d3
	WORD	$0xf3acc884	// vmlal.u32 q6,d28,d4
	WORD	$0xf3a4e883	// vmlal.u32 q7,d20,d3
	WORD	$0xf3e40885	// vmlal.u32 q8,d20,d5
	WORD	$0xf3a8a886	// vmlal.u32 q5,d24,d6
	WORD	$0xf3e62885	// vmlal.u32 q9,d22,d5
	WORD	$0xf3aac886	// vmlal.u32 q6,d26,d6
	WORD	$0xf3ace886	// vmlal.u32 q7,d28,d6
	WORD	$0xf3ec0888	// vmlal.u32 q8,d28,d8
	WORD	$0xf2300150	// vorn q0,q0,q0
	WORD	$0xf3a6a888	// vmlal.u32 q5,d22,d8
	WORD	$0xf39a00d0	// vshr.u64 q0,q0,#38
	WORD	$0xf3e42887	// vmlal.u32 q9,d20,d7
	WORD	$0xf3a8c888	// vmlal.u32 q6,d24,d8
	WORD	$0xf3aae888	// vmlal.u32 q7,d26,d8
lb_Lshort_tail:
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// horizontal addition
	WORD	$0xf27008a1	// vadd.i64 d16,d16,d17
	WORD	$0xf23aa80b	// vadd.i64 d10,d10,d11
	WORD	$0xf27228a3	// vadd.i64 d18,d18,d19
	WORD	$0xf23cc80d	// vadd.i64 d12,d12,d13
	WORD	$0xf23ee80f	// vadd.i64 d14,d14,d15
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// lazy reduction, but without narrowing
	WORD	$0xf3e6e0f0	// vshr.u64 q15,q8,#26
	WORD	$0xf24001d0	// vand.i64 q8,q8,q0
	WORD	$0xf3a680da	// vshr.u64 q4,q5,#26
	WORD	$0xf20aa150	// vand.i64 q5,q5,q0
	WORD	$0xf27228ee	// vadd.i64 q9,q9,q15
	WORD	$0xf23cc848	// vadd.i64 q6,q6,q4
	WORD	$0xf3e6e0f2	// vshr.u64 q15,q9,#26
	WORD	$0xf24221d0	// vand.i64 q9,q9,q0
	WORD	$0xf3a680dc	// vshr.u64 q4,q6,#26
	WORD	$0xf20cc150	// vand.i64 q6,q6,q0
	WORD	$0xf23ee848	// vadd.i64 q7,q7,q4
	WORD	$0xf23aa86e	// vadd.i64 q5,q5,q15
	WORD	$0xf2c2e5fe	// vshl.u64 q15,q15,#2
	WORD	$0xf3a680de	// vshr.u64 q4,q7,#26
	WORD	$0xf20ee150	// vand.i64 q7,q7,q0
	WORD	$0xf23aa86e	// vadd.i64 q5,q5,q15
	WORD	$0xf27008c8	// vadd.i64 q8,q8,q4
	WORD	$0xf3e6e0da	// vshr.u64 q15,q5,#26
	WORD	$0xf20aa150	// vand.i64 q5,q5,q0
	WORD	$0xf3a680f0	// vshr.u64 q4,q8,#26
	WORD	$0xf24001d0	// vand.i64 q8,q8,q0
	WORD	$0xf23cc86e	// vadd.i64 q6,q6,q15
	WORD	$0xf27228c8	// vadd.i64 q9,q9,q4
	CMP	$0, R2
	B.NE	lb_Leven
	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	// store hash value
	WORD	$0xf480ab4d	// vst4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]!
	WORD	$0xf4c0280f	// vst1.32 {d18[0]},[r0]
	RET
lb_LOPENSSL_armcap:

