powerpc/crypto/aes-gcm-p10.S

1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  # Accelerated AES-GCM stitched implementation for ppc64le.
5  # Copyright 2022- IBM Inc. All rights reserved
22  # Hash keys = v3 - v14
29  # v31 - counter 1
32  #     vs0 - vs14 for round keys
35  # This implementation uses stitched AES-GCM approach to improve overall performance.
48  # v15 - v18 - input states
49  # vs1 - vs9 - round keys
52 	xxlor	19+32, 1, 1
57 	vcipher	15, 15, 19
62 	vcipher	15, 15, 20
67 	vcipher	15, 15, 21
72 	vcipher	15, 15, 22
82 	vcipher	15, 15, 19
87 	vcipher	15, 15, 20
92 	vcipher	15, 15, 21
97 	vcipher	15, 15, 22
103 	vcipher	15, 15, 23
110  # v15 - v22 - input states
111  # vs1 - vs9 - round keys
114 	xxlor	23+32, 1, 1
119 	vcipher	15, 15, 23
128 	vcipher	15, 15, 24
137 	vcipher	15, 15, 25
146 	vcipher	15, 15, 26
160 	vcipher	15, 15, 23
169 	vcipher	15, 15, 24
178 	vcipher	15, 15, 25
187 	vcipher	15, 15, 26
197 	vcipher	15, 15, 23
208 	xxlor	19+32, 1, 1
213 	vcipher 15, 15, 19
214 	vcipher 15, 15, 20
215 	vcipher 15, 15, 21
216 	vcipher 15, 15, 22
223 	vcipher 15, 15, 19
224 	vcipher 15, 15, 20
225 	vcipher 15, 15, 21
226 	vcipher 15, 15, 22
229 	vcipher 15, 15, 19
236 	vxor		15, 15, 0
238 	vpmsumd		23, 12, 15		# H4.L * X.L
247 	vpmsumd		24, 13, 15		# H4.L * X.H + H4.H * X.L
267 	vpmsumd		24, 14, 15		# H4.H * X.H
290  # v15 - v22 - input blocks
294 	vxor		15, 15, 0		# Xi + X
296 	vpmsumd		23, 12, 15		# H4.L * X.L
305 	vpmsumd		24, 13, 15		# H4.L * X.H + H4.H * X.L
326 	vpmsumd		24, 14, 15		# H4.H * X.H
341 	vxor		27, 23, 27		# 1st Xi
428 	stdu 1,-640(1)
431 	std	14,112(1)
432 	std	15,120(1)
433 	std	16,128(1)
434 	std	17,136(1)
435 	std	18,144(1)
436 	std	19,152(1)
437 	std	20,160(1)
438 	std	21,168(1)
440 	stvx	20, 9, 1
442 	stvx	21, 9, 1
444 	stvx	22, 9, 1
446 	stvx	23, 9, 1
448 	stvx	24, 9, 1
450 	stvx	25, 9, 1
452 	stvx	26, 9, 1
454 	stvx	27, 9, 1
456 	stvx	28, 9, 1
458 	stvx	29, 9, 1
460 	stvx	30, 9, 1
462 	stvx	31, 9, 1
463 	stxv	14, 464(1)
464 	stxv	15, 480(1)
465 	stxv	16, 496(1)
466 	stxv	17, 512(1)
467 	stxv	18, 528(1)
468 	stxv	19, 544(1)
469 	stxv	20, 560(1)
470 	stxv	21, 576(1)
471 	stxv	22, 592(1)
472 	std	0, 656(1)
476 	lxv	14, 464(1)
477 	lxv	15, 480(1)
478 	lxv	16, 496(1)
479 	lxv	17, 512(1)
480 	lxv	18, 528(1)
481 	lxv	19, 544(1)
482 	lxv	20, 560(1)
483 	lxv	21, 576(1)
484 	lxv	22, 592(1)
486 	lvx	20, 9, 1
488 	lvx	21, 9, 1
490 	lvx	22, 9, 1
492 	lvx	23, 9, 1
494 	lvx	24, 9, 1
496 	lvx	25, 9, 1
498 	lvx	26, 9, 1
500 	lvx	27, 9, 1
502 	lvx	28, 9, 1
504 	lvx	29, 9, 1
506 	lvx	30, 9, 1
508 	lvx	31, 9, 1
510 	ld	0, 656(1)
511 	ld      14,112(1)
512 	ld      15,120(1)
513 	ld      16,128(1)
514 	ld      17,136(1)
515 	ld      18,144(1)
516 	ld      19,152(1)
517 	ld      20,160(1)
518 	ld	21,168(1)
521 	addi	1, 1, 640
528 	# load Hash - h^4, h^3, h^2, h
564  #    r3 - inp
565  #    r4 - out
566  #    r5 - len
567  #    r6 - AES round keys
568  #    r7 - iv and other data
569  #    r8 - Xi, HPoli, hash keys
581 	# initialize ICB: GHASH( IV ), IV - r7
582 	lxvb16x	30+32, 0, 7	# load IV  - v30
587 	# counter 1
589 	vspltisb 22, 1
590 	vsldoi	31, 31, 22,1	# counter 1
594 	lxv	1, 0x10(6)
605 	# load rounds - 10 (128), 12 (192), 14 (256)
611 	vxor	15, 30, 29	# IV + round key - add round key 0
640 	ld	15, 56(7)
641 	cmpdi	15, 0
644 	cmpdi	15, 16
650 	divdu	10, 12, 10	# n 128 bytes-blocks
671 	li	15, 16
683 	lxvb16x		15, 0, 14	# load block
684 	lxvb16x		16, 15, 14	# load block
703 	vcipher	15, 15, 23
712 	vcipher	15, 15, 24
729 	vcipher	15, 15, 23
738 	vcipher	15, 15, 24
757 	vcipherlast     15, 15, 23
760 	xxlxor		47, 47, 15
763 	stxvb16x        48, 15, 9	# store output
797 	vxor    15, 30, 27		# add round key
813 	addi    12, 12, -128
837 	lxvb16x 15, 0, 14		# load block
849 	vcipher	15, 15, 23
850 	vcipher	15, 15, 24
860 	vcipher	15, 15, 23
861 	vcipher	15, 15, 24
869 	vcipherlast     15, 15, 23
871 	xxlxor		47, 47, 15
876 	vmr		28, 15
879 	addi		12, 12, -16
883 	vxor		15, 30, 19		# add round key
887 	li	15, 0
888 	std	15, 56(7)		# clear partial?
905 	vcipher	15, 15, 23
906 	vcipher	15, 15, 24
916 	vcipher	15, 15, 23
917 	vcipher	15, 15, 24
925 	vcipherlast     15, 15, 23
929 	ld	15, 56(7)		# partial?
930 	cmpdi	15, 0
940 	lxvb16x	15, 0, 14		# load last block
941 	xxlxor	47, 47, 15
944 	li	15, 16
945 	sub	15, 15, 12		# index to the mask
947 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
948 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
950 	stvx	16, 10, 1
952 	stvx	17, 10, 1
954 	addi	10, 1, 192
955 	lxvb16x	16, 15, 10		# load partial block mask
958 	vmr	28, 15
976 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
977 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
979 	stxvb16x	17+32, 10, 1
981 	stxvb16x	16+32, 10, 1
983 	stxvb16x	17+32, 10, 1
985 	addi	10, 1, 192
994 	add	17, 15, 5
997 	GEN_MASK 18, 15, 5
1002 	GEN_MASK 18, 15, 16
1006 	sldi	16, 15, 3
1013 	vmr	28, 15
1031 	sldi	16, 15, 3
1033 	vslo	15, 15, 16
1034 	#stxvb16x 15+32, 0, 9		# last block
1037 	sub	17, 16, 15		# 16 - partial
1039 	add	16, 15, 5
1047 	stxvb16x	15+32, 10, 1	# save current block
1049 	addi		10, 9, -1
1050 	addi		16, 1, 191
1054         lbzu		18, 1(16)
1055 	stbu		18, 1(10)
1064 	add	15, 15, 5
1065 	cmpdi	15, 16
1071 	vxor	15, 30, 29		# IV + round key - add round key 0
1072 	li	15, 0
1073 	std	15, 56(7)		# partial done - clear
1076 	std	15, 56(7)		# partial
1083  # r9 - output
1084  # r12 - remaining bytes
1085  # v15 - partial input data
1089 	stxvb16x	15+32, 10, 1		# last block
1091 	addi		10, 9, -1
1092 	addi		16, 1, 191
1095 	li		15, 0
1098         lbzu		14, 1(16)
1099 	stbu		14, 1(10)
1122 	# initialize ICB: GHASH( IV ), IV - r7
1123 	lxvb16x	30+32, 0, 7	# load IV  - v30
1128 	# counter 1
1130 	vspltisb 22, 1
1131 	vsldoi	31, 31, 22,1	# counter 1
1135 	lxv	1, 0x10(6)
1146 	# load rounds - 10 (128), 12 (192), 14 (256)
1152 	vxor	15, 30, 29	# IV + round key - add round key 0
1181 	ld	15, 56(7)
1182 	cmpdi	15, 0
1185 	cmpdi	15, 16
1191 	divdu	10, 12, 10	# n 128 bytes-blocks
1212 	li	15, 16
1224 	lxvb16x		15, 0, 14	# load block
1225 	lxvb16x		16, 15, 14	# load block
1244 	vcipher	15, 15, 23
1253 	vcipher	15, 15, 24
1270 	vcipher	15, 15, 23
1279 	vcipher	15, 15, 24
1298 	vcipherlast     15, 15, 23
1301 	xxlxor		47, 47, 15
1304 	stxvb16x        48, 15, 9	# store output
1332 	xxlor           15+32, 15, 15
1347 	vxor    15, 30, 27		# add round key
1363 	addi    12, 12, -128
1387 	lxvb16x 15, 0, 14		# load block
1399 	vcipher	15, 15, 23
1400 	vcipher	15, 15, 24
1410 	vcipher	15, 15, 23
1411 	vcipher	15, 15, 24
1419 	vcipherlast     15, 15, 23
1421 	xxlxor		47, 47, 15
1426 	xxlor           28+32, 15, 15
1427 	#vmr		28, 15
1430 	addi		12, 12, -16
1434 	vxor		15, 30, 19		# add round key
1438 	li	15, 0
1439 	std	15, 56(7)		# clear partial?
1456 	vcipher	15, 15, 23
1457 	vcipher	15, 15, 24
1467 	vcipher	15, 15, 23
1468 	vcipher	15, 15, 24
1476 	vcipherlast     15, 15, 23
1479 	li	21, 1			# decrypt
1480 	ld	15, 56(7)		# partial?
1481 	cmpdi	15, 0
1490 	lxvb16x	15, 0, 14		# load last block
1491 	xxlxor	47, 47, 15
1494 	li	15, 16
1495 	sub	15, 15, 12		# index to the mask
1497 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
1498 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
1500 	stvx	16, 10, 1
1502 	stvx	17, 10, 1
1504 	addi	10, 1, 192
1505 	lxvb16x	16, 15, 10		# load partial block mask
1508 	xxland	32+28, 15, 16
1509 	#vmr	28, 15