x86/crypto/aes_ctrby8_avx-x86_64.S

1 /* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
19  *    https://github.com/intel/intel-ipsec-mb
58 #define	XDATA		1
59 #define KEY_128		1
64 .align 16
118 		vmovdqa	0*16(p_keys), xkey0
126 			vpaddq	(ddq_add_1 + 16 * i)(%rip), xtmp, var_xdata
127 			.set i, (i +1)
133 			.set i, (i +1)
137 		.set i, 1
138 		.rept (by - 1)
140 			vpaddq	(ddq_add_1 + 16 * (i - 1))(%rip), xcounter, var_xdata
142 			jnz 1f
145 			1:
147 			.set i, (i +1)
151 	vmovdqa	1*16(p_keys), xkeyA
157 		vpaddq	(ddq_add_1 + 16 * (by - 1))(%rip), xcounter, xcounter
159 		jnz	1f
161 		1:
164 	.set i, 1
165 	.rept (by - 1)
168 		.set i, (i +1)
171 	vmovdqa	2*16(p_keys), xkeyB
176 		vaesenc	xkeyA, var_xdata, var_xdata		/* key 1 */
177 		.set i, (i +1)
182 			vmovdqa	3*16(p_keys), xkey4
185 		vmovdqa	3*16(p_keys), xkeyA
192 		.set i, (i +1)
195 	add	$(16*by), p_in
198 		vmovdqa	4*16(p_keys), xkeyB
201 			vmovdqa	4*16(p_keys), xkey4
214 		.set i, (i +1)
217 	vmovdqa	5*16(p_keys), xkeyA
228 		.set i, (i +1)
233 			vmovdqa	6*16(p_keys), xkey8
236 		vmovdqa	6*16(p_keys), xkeyB
243 		.set i, (i +1)
246 	vmovdqa	7*16(p_keys), xkeyA
257 		.set i, (i +1)
261 		vmovdqa	8*16(p_keys), xkeyB
264 			vmovdqa	8*16(p_keys), xkey8
272 		.set i, (i +1)
277 			vmovdqa	9*16(p_keys), xkey12
280 		vmovdqa	9*16(p_keys), xkeyA
292 		.set i, (i +1)
295 	vmovdqa	10*16(p_keys), xkeyB
306 		.set i, (i +1)
310 		vmovdqa	11*16(p_keys), xkeyA
322 		.set i, (i +1)
327 			vmovdqa	12*16(p_keys), xkey12
334 			.set i, (i +1)
338 			vmovdqa	13*16(p_keys), xkeyA
350 			.set i, (i +1)
354 			vmovdqa	14*16(p_keys), xkeyB
361 				.set i, (i +1)
369 				.set i, (i +1)
376 		.set j, (i+1)
377 		VMOVDQ	(i*16 - 16*by)(p_in), xkeyA
378 		VMOVDQ	(j*16 - 16*by)(p_in), xkeyB
387 		VMOVDQ	(i*16 - 16*by)(p_in), xkeyA
395 		VMOVDQ	var_xdata, i*16(p_out)
396 		.set i, (i+1)
401 	do_aes \val, 1, \key_len, \xctr
411 	cmp	$16, num_bytes
424 	and	$(7*16), tmp
427 	/* 1 <= tmp <= 7 */
428 	cmp	$(4*16), tmp
433 	cmp	$(2*16), tmp
438 	do_aes_load	1, \key_len, \xctr
439 	add	$(1*16), p_out
440 	and	$(~7*16), num_bytes
446 	add	$(2*16), p_out
447 	and	$(~7*16), num_bytes
454 	add	$(3*16), p_out
455 	and	$(~7*16), num_bytes
461 	add	$(4*16), p_out
462 	and	$(~7*16), num_bytes
467 	cmp	$(6*16), tmp
473 	add	$(5*16), p_out
474 	and	$(~7*16), num_bytes
480 	add	$(6*16), p_out
481 	and	$(~7*16), num_bytes
487 	add	$(7*16), p_out
488 	and	$(~7*16), num_bytes
494 		vmovdqa	0*16(p_keys), xkey0
495 		vmovdqa	4*16(p_keys), xkey4
496 		vmovdqa	8*16(p_keys), xkey8
497 		vmovdqa	12*16(p_keys), xkey12
499 		vmovdqa	0*16(p_keys), xkey0
500 		vmovdqa	3*16(p_keys), xkey4
501 		vmovdqa	6*16(p_keys), xkey8
502 		vmovdqa	9*16(p_keys), xkey12
504 .align 16
508 	add	$(8*16), p_out
509 	sub	$(8*16), num_bytes
569 	do_aes_ctrmain KEY_128 1
582 	do_aes_ctrmain KEY_192 1
595 	do_aes_ctrmain KEY_256 1