powerpc/crypto/aes-gcm-p10.S

1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  # Accelerated AES-GCM stitched implementation for ppc64le.
5  # Copyright 2022- IBM Inc. All rights reserved
22  # Hash keys = v3 - v14
29  # v31 - counter 1
32  #     vs0 - vs14 for round keys
35  # This implementation uses stitched AES-GCM approach to improve overall performance.
48  # v15 - v18 - input states
49  # vs1 - vs9 - round keys
57 	vcipher	15, 15, 19
62 	vcipher	15, 15, 20
67 	vcipher	15, 15, 21
72 	vcipher	15, 15, 22
82 	vcipher	15, 15, 19
87 	vcipher	15, 15, 20
92 	vcipher	15, 15, 21
97 	vcipher	15, 15, 22
103 	vcipher	15, 15, 23
110  # v15 - v22 - input states
111  # vs1 - vs9 - round keys
119 	vcipher	15, 15, 23
128 	vcipher	15, 15, 24
137 	vcipher	15, 15, 25
146 	vcipher	15, 15, 26
160 	vcipher	15, 15, 23
169 	vcipher	15, 15, 24
178 	vcipher	15, 15, 25
187 	vcipher	15, 15, 26
197 	vcipher	15, 15, 23
213 	vcipher 15, 15, 19
214 	vcipher 15, 15, 20
215 	vcipher 15, 15, 21
216 	vcipher 15, 15, 22
223 	vcipher 15, 15, 19
224 	vcipher 15, 15, 20
225 	vcipher 15, 15, 21
226 	vcipher 15, 15, 22
229 	vcipher 15, 15, 19
236 	vxor		15, 15, 0
238 	vpmsumd		23, 12, 15		# H4.L * X.L
247 	vpmsumd		24, 13, 15		# H4.L * X.H + H4.H * X.L
267 	vpmsumd		24, 14, 15		# H4.H * X.H
290  # v15 - v22 - input blocks
294 	vxor		15, 15, 0		# Xi + X
296 	vpmsumd		23, 12, 15		# H4.L * X.L
305 	vpmsumd		24, 13, 15		# H4.L * X.H + H4.H * X.L
326 	vpmsumd		24, 14, 15		# H4.H * X.H
400 	vxor		28, 28, 0
423 	vmr		0, 22			# update hash
428 	stdu 1,-640(1)
429 	mflr 0
432 	std	15,120(1)
464 	stxv	15, 480(1)
472 	std	0, 656(1)
477 	lxv	15, 480(1)
510 	ld	0, 656(1)
512 	ld      15,120(1)
520 	mtlr	0
526 	lxvb16x	32, 0, 8	# load Xi
528 	# load Hash - h^4, h^3, h^2, h
564  #    r3 - inp
565  #    r4 - out
566  #    r5 - len
567  #    r6 - AES round keys
568  #    r7 - iv and other data
569  #    r8 - Xi, HPoli, hash keys
572  #    Xi is at 0 in gcm_table (Xip).
581 	# initialize ICB: GHASH( IV ), IV - r7
582 	lxvb16x	30+32, 0, 7	# load IV  - v30
585 	li	11, 0		# block index
593 	lxv	0, 0(6)
594 	lxv	1, 0x10(6)
595 	lxv	2, 0x20(6)
596 	lxv	3, 0x30(6)
597 	lxv	4, 0x40(6)
598 	lxv	5, 0x50(6)
599 	lxv	6, 0x60(6)
600 	lxv	7, 0x70(6)
601 	lxv	8, 0x80(6)
602 	lxv	9, 0x90(6)
603 	lxv	10, 0xa0(6)
605 	# load rounds - 10 (128), 12 (192), 14 (256)
610 	xxlor	32+29, 0, 0
611 	vxor	15, 30, 29	# IV + round key - add round key 0
617 	lxv	11, 0xb0(6)
618 	lxv	12, 0xc0(6)
624 	lxv	13, 0xd0(6)
625 	lxv	14, 0xe0(6)
640 	ld	15, 56(7)
641 	cmpdi	15, 0
644 	cmpdi	15, 16
650 	divdu	10, 12, 10	# n 128 bytes-blocks
651 	cmpdi	10, 0
671 	li	15, 16
683 	lxvb16x		15, 0, 14	# load block
684 	lxvb16x		16, 15, 14	# load block
703 	vcipher	15, 15, 23
712 	vcipher	15, 15, 24
729 	vcipher	15, 15, 23
738 	vcipher	15, 15, 24
757 	vcipherlast     15, 15, 23
760 	xxlxor		47, 47, 15
761 	stxvb16x        47, 0, 9	# store output
763 	stxvb16x        48, 15, 9	# store output
794 	xxlor	27+32, 0, 0
797 	vxor    15, 30, 27		# add round key
813 	addi    12, 12, -128
819 	stxvb16x 30+32, 0, 7		# update IV
822 	cmpdi   12, 0
837 	lxvb16x 15, 0, 14		# load block
849 	vcipher	15, 15, 23
850 	vcipher	15, 15, 24
860 	vcipher	15, 15, 23
861 	vcipher	15, 15, 24
869 	vcipherlast     15, 15, 23
871 	xxlxor		47, 47, 15
872 	stxvb16x	47, 0, 9	# store output
876 	vmr		28, 15
879 	addi		12, 12, -16
881 	xxlor		19+32, 0, 0
883 	vxor		15, 30, 19		# add round key
887 	li	15, 0
888 	std	15, 56(7)		# clear partial?
889 	stxvb16x 30+32, 0, 7		# update IV
890 	cmpdi	12, 0
905 	vcipher	15, 15, 23
906 	vcipher	15, 15, 24
916 	vcipher	15, 15, 23
917 	vcipher	15, 15, 24
925 	vcipherlast     15, 15, 23
928 	li	21, 0			# encrypt
929 	ld	15, 56(7)		# partial?
930 	cmpdi	15, 0
934 	cmpdi	12, 0
940 	lxvb16x	15, 0, 14		# load last block
941 	xxlxor	47, 47, 15
944 	li	15, 16
945 	sub	15, 15, 12		# index to the mask
947 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
948 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
955 	lxvb16x	16, 15, 10		# load partial block mask
958 	vmr	28, 15
964 	stxvb16x 30+32, 0, 7		# update IV
968 	stxvb16x	32, 0, 8		# write out Xi
976 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
977 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
986 	lxvb16x	\_mask, 0, 10		# load partial block mask
994 	add	17, 15, 5
997 	GEN_MASK 18, 15, 5
1002 	GEN_MASK 18, 15, 16
1005 	lxvb16x	17+32, 0, 14		# load last block
1006 	sldi	16, 15, 3
1007 	mtvsrdd	32+16, 0, 16
1012 	vxor	0, 0, 0			# clear Xi
1013 	vmr	28, 15
1015 	cmpdi	21, 0			# encrypt/decrypt ops?
1025 	vxor	0, 0, 29
1026 	stxvb16x 32, 0, 8		# save Xi
1031 	sldi	16, 15, 3
1032 	mtvsrdd	32+16, 0, 16
1033 	vslo	15, 15, 16
1034 	#stxvb16x 15+32, 0, 9		# last block
1037 	sub	17, 16, 15		# 16 - partial
1039 	add	16, 15, 5
1047 	stxvb16x	15+32, 10, 1	# save current block
1049 	addi		10, 9, -1
1064 	add	15, 15, 5
1065 	cmpdi	15, 16
1069 	stxvb16x 30+32, 0, 7		# update IV
1070 	xxlor	32+29, 0, 0
1071 	vxor	15, 30, 29		# IV + round key - add round key 0
1072 	li	15, 0
1073 	std	15, 56(7)		# partial done - clear
1076 	std	15, 56(7)		# partial
1083  # r9 - output
1084  # r12 - remaining bytes
1085  # v15 - partial input data
1089 	stxvb16x	15+32, 10, 1		# last block
1091 	addi		10, 9, -1
1095 	li		15, 0
1106 	stxvb16x	32, 0, 8		# write out Xi
1122 	# initialize ICB: GHASH( IV ), IV - r7
1123 	lxvb16x	30+32, 0, 7	# load IV  - v30
1126 	li	11, 0		# block index
1134 	lxv	0, 0(6)
1135 	lxv	1, 0x10(6)
1136 	lxv	2, 0x20(6)
1137 	lxv	3, 0x30(6)
1138 	lxv	4, 0x40(6)
1139 	lxv	5, 0x50(6)
1140 	lxv	6, 0x60(6)
1141 	lxv	7, 0x70(6)
1142 	lxv	8, 0x80(6)
1143 	lxv	9, 0x90(6)
1144 	lxv	10, 0xa0(6)
1146 	# load rounds - 10 (128), 12 (192), 14 (256)
1151 	xxlor	32+29, 0, 0
1152 	vxor	15, 30, 29	# IV + round key - add round key 0
1158 	lxv	11, 0xb0(6)
1159 	lxv	12, 0xc0(6)
1165 	lxv	13, 0xd0(6)
1166 	lxv	14, 0xe0(6)
1181 	ld	15, 56(7)
1182 	cmpdi	15, 0
1185 	cmpdi	15, 16
1191 	divdu	10, 12, 10	# n 128 bytes-blocks
1192 	cmpdi	10, 0
1212 	li	15, 16
1224 	lxvb16x		15, 0, 14	# load block
1225 	lxvb16x		16, 15, 14	# load block
1244 	vcipher	15, 15, 23
1253 	vcipher	15, 15, 24
1270 	vcipher	15, 15, 23
1279 	vcipher	15, 15, 24
1298 	vcipherlast     15, 15, 23
1301 	xxlxor		47, 47, 15
1302 	stxvb16x        47, 0, 9	# store output
1304 	stxvb16x        48, 15, 9	# store output
1332 	xxlor           15+32, 15, 15
1344 	xxlor	27+32, 0, 0
1347 	vxor    15, 30, 27		# add round key
1363 	addi    12, 12, -128
1369 	stxvb16x 30+32, 0, 7		# update IV
1372 	cmpdi   12, 0
1387 	lxvb16x 15, 0, 14		# load block
1399 	vcipher	15, 15, 23
1400 	vcipher	15, 15, 24
1410 	vcipher	15, 15, 23
1411 	vcipher	15, 15, 24
1419 	vcipherlast     15, 15, 23
1421 	xxlxor		47, 47, 15
1422 	stxvb16x	47, 0, 9	# store output
1426 	xxlor           28+32, 15, 15
1427 	#vmr		28, 15
1430 	addi		12, 12, -16
1432 	xxlor		19+32, 0, 0
1434 	vxor		15, 30, 19		# add round key
1438 	li	15, 0
1439 	std	15, 56(7)		# clear partial?
1440 	stxvb16x 30+32, 0, 7		# update IV
1441 	cmpdi	12, 0
1456 	vcipher	15, 15, 23
1457 	vcipher	15, 15, 24
1467 	vcipher	15, 15, 23
1468 	vcipher	15, 15, 24
1476 	vcipherlast     15, 15, 23
1480 	ld	15, 56(7)		# partial?
1481 	cmpdi	15, 0
1484 	cmpdi	12, 0
1490 	lxvb16x	15, 0, 14		# load last block
1491 	xxlxor	47, 47, 15
1494 	li	15, 16
1495 	sub	15, 15, 12		# index to the mask
1497 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
1498 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
1505 	lxvb16x	16, 15, 10		# load partial block mask
1508 	xxland	32+28, 15, 16
1509 	#vmr	28, 15
1515 	stxvb16x 30+32, 0, 7		# update IV
1519 	stxvb16x	32, 0, 8		# write out Xi