powerpc/crypto/aes-gcm-p10.S

1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  # Accelerated AES-GCM stitched implementation for ppc64le.
5  # Copyright 2022- IBM Inc. All rights reserved
22  # Hash keys = v3 - v14
29  # v31 - counter 1
32  #     vs0 - vs14 for round keys
35  # This implementation uses stitched AES-GCM approach to improve overall performance.
48  # v15 - v18 - input states
49  # vs1 - vs9 - round keys
78 	xxlor	20+32, 6, 6
110  # v15 - v22 - input states
111  # vs1 - vs9 - round keys
156 	xxlor	24+32, 6, 6
219 	xxlor	20+32, 6, 6
240 	vpmsumd		25, 6, 17
267 	vpmsumd		24, 14, 15		# H4.H * X.H
290  # v15 - v22 - input blocks
298 	vpmsumd		25, 6, 17
326 	vpmsumd		24, 14, 15		# H4.H * X.H
345 	vpmsumd		25, 6, 21
375 	vpmsumd		24, 14, 19		# H4.H * X.H
428 	stdu 1,-640(1)
431 	std	14,112(1)
463 	stxv	14, 464(1)
476 	lxv	14, 464(1)
511 	ld      14,112(1)
528 	# load Hash - h^4, h^3, h^2, h
539 	lxvd2x	6+32, 10, 8	# H^2l
557 	lxvd2x	14+32, 10, 8	# H^4h
564  #    r3 - inp
565  #    r4 - out
566  #    r5 - len
567  #    r6 - AES round keys
568  #    r7 - iv and other data
569  #    r8 - Xi, HPoli, hash keys
581 	# initialize ICB: GHASH( IV ), IV - r7
582 	lxvb16x	30+32, 0, 7	# load IV  - v30
593 	lxv	0, 0(6)
594 	lxv	1, 0x10(6)
595 	lxv	2, 0x20(6)
596 	lxv	3, 0x30(6)
597 	lxv	4, 0x40(6)
598 	lxv	5, 0x50(6)
599 	lxv	6, 0x60(6)
600 	lxv	7, 0x70(6)
601 	lxv	8, 0x80(6)
602 	lxv	9, 0x90(6)
603 	lxv	10, 0xa0(6)
605 	# load rounds - 10 (128), 12 (192), 14 (256)
606 	lwz	9,240(6)
611 	vxor	15, 30, 29	# IV + round key - add round key 0
617 	lxv	11, 0xb0(6)
618 	lxv	12, 0xc0(6)
624 	lxv	13, 0xd0(6)
625 	lxv	14, 0xe0(6)
626 	cmpdi	9, 14
633 	mr	14, 3
650 	divdu	10, 12, 10	# n 128 bytes-blocks
679 	lwz	10, 240(6)
683 	lxvb16x		15, 0, 14	# load block
684 	lxvb16x		16, 15, 14	# load block
685 	lxvb16x		17, 16, 14	# load block
686 	lxvb16x		18, 17, 14	# load block
687 	lxvb16x		19, 18, 14	# load block
688 	lxvb16x		20, 19, 14	# load block
689 	lxvb16x		21, 20, 14	# load block
690 	lxvb16x		22, 21, 14	# load block
691 	addi		14, 14, 128
747 	xxlor	23+32, 14, 14
749 	cmpdi	10, 14
813 	addi    12, 12, -128
831 	lwz	10, 240(6)
837 	lxvb16x 15, 0, 14		# load block
863 	xxlor	23+32, 14, 14
865 	cmpdi	10, 14
873 	addi		14, 14, 16
879 	addi		12, 12, -16
894 	lwz	10, 240(6)
919 	xxlor	23+32, 14, 14
921 	cmpdi	10, 14
940 	lxvb16x	15, 0, 14		# load last block
947 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
948 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
976 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
977 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
1005 	lxvb16x	17+32, 0, 14		# load last block
1037 	sub	17, 16, 15		# 16 - partial
1049 	addi		10, 9, -1
1059 	add	14, 14, 17
1071 	vxor	15, 30, 29		# IV + round key - add round key 0
1073 	std	15, 56(7)		# partial done - clear
1083  # r9 - output
1084  # r12 - remaining bytes
1085  # v15 - partial input data
1091 	addi		10, 9, -1
1098         lbzu		14, 1(16)
1099 	stbu		14, 1(10)
1122 	# initialize ICB: GHASH( IV ), IV - r7
1123 	lxvb16x	30+32, 0, 7	# load IV  - v30
1134 	lxv	0, 0(6)
1135 	lxv	1, 0x10(6)
1136 	lxv	2, 0x20(6)
1137 	lxv	3, 0x30(6)
1138 	lxv	4, 0x40(6)
1139 	lxv	5, 0x50(6)
1140 	lxv	6, 0x60(6)
1141 	lxv	7, 0x70(6)
1142 	lxv	8, 0x80(6)
1143 	lxv	9, 0x90(6)
1144 	lxv	10, 0xa0(6)
1146 	# load rounds - 10 (128), 12 (192), 14 (256)
1147 	lwz	9,240(6)
1152 	vxor	15, 30, 29	# IV + round key - add round key 0
1158 	lxv	11, 0xb0(6)
1159 	lxv	12, 0xc0(6)
1165 	lxv	13, 0xd0(6)
1166 	lxv	14, 0xe0(6)
1167 	cmpdi	9, 14
1174 	mr	14, 3
1191 	divdu	10, 12, 10	# n 128 bytes-blocks
1220 	lwz	10, 240(6)
1224 	lxvb16x		15, 0, 14	# load block
1225 	lxvb16x		16, 15, 14	# load block
1226 	lxvb16x		17, 16, 14	# load block
1227 	lxvb16x		18, 17, 14	# load block
1228 	lxvb16x		19, 18, 14	# load block
1229 	lxvb16x		20, 19, 14	# load block
1230 	lxvb16x		21, 20, 14	# load block
1231 	lxvb16x		22, 21, 14	# load block
1232 	addi		14, 14, 128
1288 	xxlor	23+32, 14, 14
1290 	cmpdi	10, 14
1363 	addi    12, 12, -128
1381 	lwz	10, 240(6)
1387 	lxvb16x 15, 0, 14		# load block
1413 	xxlor	23+32, 14, 14
1415 	cmpdi	10, 14
1423 	addi		14, 14, 16
1430 	addi		12, 12, -16
1445 	lwz	10, 240(6)
1470 	xxlor	23+32, 14, 14
1472 	cmpdi	10, 14
1490 	lxvb16x	15, 0, 14		# load last block
1497 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
1498 	vspltisb	17, 0		# second 16 bytes - 0x0000...00