arm64/crypto/ghash-ce-core.S

1 /* SPDX-License-Identifier: GPL-2.0-only */
5  * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
61 	.arch		armv8-a+crypto
72 	ext		t3.8b, \ad\().8b, \ad\().8b, #1		// A1
73 	ext		t5.8b, \ad\().8b, \ad\().8b, #2		// A2
74 	ext		t7.8b, \ad\().8b, \ad\().8b, #3		// A3
106 	pmull\t		t9.8h, \ad, \b4\().\nb			// K = A*B4
124 	// t9 = (K) (P6 + P7) << 32
136 	ext		t3.16b, t3.16b, t3.16b, #15
137 	ext		t5.16b, t5.16b, t5.16b, #14
138 	ext		t7.16b, t7.16b, t7.16b, #13
139 	ext		t9.16b, t9.16b, t9.16b, #12
149 	ld1		{HH.2d-HH4.2d}, [x8]
164 	ext		SHASH2.16b, SHASH.16b, SHASH.16b, #8
171 	ushr		k00_16.2d, k32_48.2d, #32
190 	ext		ss1.8b, SHASH2.8b, SHASH2.8b, #1
191 	ext		ss2.8b, SHASH2.8b, SHASH2.8b, #2
192 	ext		ss3.8b, SHASH2.8b, SHASH2.8b, #3
193 	ext		ss4.8b, SHASH2.8b, SHASH2.8b, #4
197 	// PMULL (64x64->128) based reduction for CPUs that can do
208 	ext		T2.16b, XL.16b, XL.16b, #8
214 	// 64x64->128 PMULL instruction
227 	ext		T1.16b, XL.16b, XH.16b, #8
256 1:	ld1		{XM3.16b-TT4.16b}, [x2], #64
265 	ext		IN1.16b, TT4.16b, TT4.16b, #8
266 	ext		XL3.16b, TT3.16b, TT3.16b, #8
278 	ext		IN1.16b, T2.16b, T2.16b, #8
292 	ext		IN1.16b, T1.16b, T1.16b, #8
293 	ext		TT3.16b, XL.16b, XL.16b, #8
307 	ext		T1.16b, XL.16b, XH.16b, #8
325 	ext		T2.16b, XL.16b, XL.16b, #8
326 	ext		IN1.16b, T1.16b, T1.16b, #8
336 	ext		T1.16b, XL.16b, XH.16b, #8
352 	 *			   struct ghash_key const *k, const char *head)
388 	ld1		{K0.4s-K3.4s}, [\rk]
389 	ld1		{K4.4s-K5.4s}, [\tmp]
391 	sub		\tmp, \tmp, #32
392 	ld1		{KK.4s-KM.4s}, [\tmp]
409 	ld1		{K6.4s-K7.4s}, [\tmp], #32
426 	ld1		{K8.4s-K9.4s}, [\tmp], #32
429 	ld1		{K6.4s-K7.4s}, [\tmp]
444 	ld1		{HH.2d-HH4.2d}, [x3]
470 	ld1		{INP0.16b-INP3.16b}, [x2], #64
500 	cmp		x0, #-16
502 	cmp		x0, #-32
504 	cmp		x0, #-48
524 	st1		{INP0.16b-INP3.16b}, [x1], #64
543 	ext		XL.16b, XL.16b, XL.16b, #8
557 	mvn		XL.16b, XL.16b			// -1 for fail, 0 for pass
572 6:	ld1		{T1.16b-T2.16b}, [x17], #32	// permute vectors
593 	tbl		INP3.16b, {INP3.16b}, T1.16b	// clear non-data bits
601 	 *			  struct ghash_key const *k, u64 dg[], u8 ctr[],
610 	 *			  struct ghash_key const *k, u64 dg[], u8 ctr[],
626 	ext		XL.16b, XL.16b, XL.16b, #8
638 	ext		T1.16b, T2.16b, T2.16b, #8
642 	ext		T2.16b, TT3.16b, TT3.16b, #8
646 	ext		IN1.16b, TT4.16b, TT4.16b, #8
651 	ext		IN1.16b, T1.16b, T1.16b, #8
658 	ext		T1.16b, T2.16b, T2.16b, #8
668 	ext		T2.16b, TT3.16b, TT3.16b, #8
678 	ext		IN1.16b, TT4.16b, TT4.16b, #8
689 	ext		T1.16b, XL.16b, XH.16b, #8
719 	ld1		{K6.4s-K7.4s}, [x10], #32
727 	ld1		{K8.4s-K9.4s}, [x10], #32
731 	ld1		{K6.4s-K7.4s}, [x10]