arm64/crypto/aes-modes.S

1 /* SPDX-License-Identifier: GPL-2.0-only */
3  * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
5  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
8 /* included by aes-ce.S and aes-neon.S */
49 	 *		   int blocks)
51 	 *		   int blocks)
62 	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
66 	st1		{v0.16b-v3.16b}, [x0], #64
92 	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
96 	st1		{v0.16b-v3.16b}, [x0], #64
116 	 *		   int blocks, u8 iv[])
118 	 *		   int blocks, u8 iv[])
120 	 *			 int rounds, int blocks, u8 iv[],
123 	 *			 int rounds, int blocks, u8 iv[],
130 	mov		w8, #14				/* AES-256: 14 rounds */
143 	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
152 	st1		{v0.16b-v3.16b}, [x0], #64
174 	mov		w8, #14				/* AES-256: 14 rounds */
188 	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
195 	sub		x1, x1, #32
208 	sub		x1, x1, #16
215 	st1		{v0.16b-v3.16b}, [x0], #64
247 	sub		x4, x4, #16
250 	sub		x9, x9, x4
276 	sub		x4, x4, #16
279 	sub		x9, x9, x4
330 	BLOCKS		.req x13
341 	 * the 64-bit counter with the IV.
353 	sub		BYTES_W, BYTES_W, #MAX_STRIDE << 4
360 	 * Set up the counter values in v0-v{MAX_STRIDE-1}.
362 	 * If we are encrypting less than MAX_STRIDE blocks, the tail block
364 	 * v{MAX_STRIDE-1}.  For example: if encrypting two blocks with
365 	 * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks.
368 		add		CTR, CTR, BLOCKS
370 		adds		IV_PART, IV_PART, BLOCKS
378 		sub		x6, CTR, #MAX_STRIDE - 1
379 		sub		x7, CTR, #MAX_STRIDE - 2
380 		sub		x8, CTR, #MAX_STRIDE - 3
381 		sub		x9, CTR, #MAX_STRIDE - 4
382 ST5(		sub		x10, CTR, #MAX_STRIDE - 5	)
413 		 * Apply carry to counter blocks if needed.
417 		 * many counter blocks need to be updated.
421 		sub		x16, x16, IV_PART, lsl #3
438 		sub		x7, IV_PART, #MAX_STRIDE - 1
439 		sub		x8, IV_PART, #MAX_STRIDE - 2
440 		sub		x9, IV_PART, #MAX_STRIDE - 3
445 ST5(		sub		x10, IV_PART, #MAX_STRIDE - 4	)
453 	 * If there are at least MAX_STRIDE blocks left, XOR the data with
457 	ld1		{v5.16b-v7.16b}, [IN], #48
463 ST5(	ld1		{v5.16b-v6.16b}, [IN], #32	)
467 	st1		{v0.16b-v3.16b}, [OUT], #64
481 	 * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext
483 	 * This code expects the last keystream block to be in v{MAX_STRIDE-1}.
484 	 * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and
485 	 * v4 should have the next two counter blocks.
488 	 * regions of memory.  Any invalid ciphertext blocks get overwritten by
489 	 * correctly computed blocks.  This approach greatly simplifies the
496 ST5(	cmp		BYTES_W, #64 - (MAX_STRIDE << 4))
498 	cmp		BYTES_W, #48 - (MAX_STRIDE << 4)
500 	cmp		BYTES_W, #32 - (MAX_STRIDE << 4)
502 	cmp		BYTES_W, #16 - (MAX_STRIDE << 4)
555 	sub		x8, x7, #16
563 	ld1		{v10.16b-v11.16b}, [x9]
583 	.unreq BLOCKS
594 	 * to be at the end of this 16-byte temporary buffer rather than the
609 	 * to be at the end of this 16-byte temporary buffer rather than the
659 	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
672 	st1		{v0.16b-v3.16b}, [x0], #64
703 	sub		x0, x0, #16
711 	sub		x9, x9, x4
729 	sub		w8, w4, #0x10
751 	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
764 	st1		{v0.16b-v3.16b}, [x0], #64
797 	sub		x9, x9, x4
820 	 *		  int blocks, u8 dg[], int enc_before, int enc_after)
832 	ld1		{v1.16b-v4.16b}, [x0], #64	/* get next pt block */