Lines Matching +full:in7 +full:- +full:in6
2 # SPDX-License-Identifier: GPL-2.0
12 # Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
58 # The module is endian-agnostic in sense that it supports both big-
59 # and little-endian cases. Data alignment in parallelizable modes is
64 # is aligned programmatically, which in turn guarantees exception-
72 # Add XTS subroutine, 9x on little- and 12x improvement on big-endian
76 # Current large-block performance in cycles per byte processed with
77 # 128-bit key (less is better).
79 # CBC en-/decrypt CTR XTS
106 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
107 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
108 die "can't locate ppc-xlate.pl";
140 addi $ptr,$ptr,-0x58
152 li $ptr,-1
154 beq- Lenc_key_abort # if ($inp==0) return -1;
156 beq- Lenc_key_abort # if ($out==0) return -1;
157 li $ptr,-2
159 blt- Lenc_key_abort
161 bgt- Lenc_key_abort
163 bne- Lenc_key_abort
190 vspltisb $outmask,-1
202 vperm $key,$in0,$in0,$mask # rotate-n-splat
222 vperm $key,$in0,$in0,$mask # rotate-n-splat
239 vperm $key,$in0,$in0,$mask # rotate-n-splat
280 vperm $key,$in1,$in1,$mask # roate-n-splat
300 vperm $key,$in1,$in1,$mask # rotate-n-splat
354 vperm $key,$in1,$in1,$mask # rotate-n-splat
406 .size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
409 $STU $sp,-$FRAME($sp)
416 bne- Ldec_key_abort
439 stw r9, -16($inp)
440 stw r10,-12($inp)
441 stw r11,-8($inp)
442 stw r12,-4($inp)
452 .size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
456 {{{ # Single block en- and decrypt procedures #
509 vspltisb v2,-1
527 .size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
534 {{{ # CBC en- and decrypt procedures #
542 bltlr-
569 vspltisb $outmask,-1
584 subi $len,$len,16 # len-=16
632 subi $len,$len,16 # len-=16
672 addi $out,$out,-1
680 vspltisb $outmask,-1
703 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
705 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
706 # v26-v31 last 6 round keys
712 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
737 li r0,-1
738 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
754 subi $rounds,$rounds,3 # -4 in total
769 stvx v24,$x00,$key_ # off-load round[1]
772 stvx v25,$x10,$key_ # off-load round[2]
779 stvx v24,$x00,$key_ # off-load round[3]
782 stvx v25,$x10,$key_ # off-load round[4]
793 lvx v24,$x00,$key_ # pre-load round[1]
795 lvx v25,$x10,$key_ # pre-load round[2]
817 lvx_u $in6,$x60,$inp
820 lvx_u $in7,$x70,$inp
824 le?vperm $in6,$in6,$in6,$inpperm
826 le?vperm $in7,$in7,$in7,$inpperm
828 vxor $out6,$in6,$rndkey0
829 vxor $out7,$in7,$rndkey0
857 subic $len,$len,128 # $len-=128
867 subfe. r0,r0,r0 # borrow?-1:0
889 # loop inX-in7 are loaded
909 lvx v24,$x00,$key_ # re-pre-load round[1]
919 lvx v25,$x10,$key_ # re-pre-load round[2]
936 vxor $in6,$in6,v31
955 vncipherlast $out7,$out7,$in6
957 lvx_u $in6,$x60,$inp
958 vmr $ivec,$in7
960 lvx_u $in7,$x70,$inp
966 le?vperm $in6,$in6,$in6,$inpperm
970 le?vperm $in7,$in7,$in7,$inpperm
986 vxor $out6,$in6,$rndkey0
989 vxor $out7,$in7,$rndkey0
992 beq Loop_cbc_dec8x # did $len-=128 borrow?
1081 vxor $in6,$in6,v31
1103 vncipherlast $out7,$out7,$in6
1104 vmr $ivec,$in7
1130 vncipherlast $out7,$out7,$in6
1131 vmr $ivec,$in7
1154 vncipherlast $out7,$out7,$in6
1155 vmr $ivec,$in7
1175 vncipherlast $out7,$out7,$in6
1176 vmr $ivec,$in7
1193 vncipherlast $out7,$out7,$in6
1194 vmr $ivec,$in7
1208 vncipherlast $out7,$out7,$in6
1209 vmr $ivec,$in7
1221 vmr $ivec,$in7
1284 .size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1293 # This code is written as 'ctr32', based on a 32-bit counter used
1294 # upstream. The kernel does *not* use a 32-bit counter. The kernel uses
1295 # a 128-bit counter.
1303 # 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
1304 # 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
1317 bltlr-
1352 vspltisb $outmask,-1
1379 vadduqm $ivec,$ivec,$one # Kernel change for 128-bit
1383 subic. $len,$len,1 # blocks--
1409 addi $out,$out,-1
1424 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1426 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1427 # v26-v31 last 6 round keys
1434 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1459 li r0,-1
1460 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1476 subi $rounds,$rounds,3 # -4 in total
1490 stvx v24,$x00,$key_ # off-load round[1]
1493 stvx v25,$x10,$key_ # off-load round[2]
1500 stvx v24,$x00,$key_ # off-load round[3]
1503 stvx v25,$x10,$key_ # off-load round[4]
1514 lvx v24,$x00,$key_ # pre-load round[1]
1516 lvx v25,$x10,$key_ # pre-load round[2]
1523 vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit)
1570 subic r11,$len,256 # $len-256, borrow $key_
1580 subfe r0,r0,r0 # borrow?-1:0
1600 lvx v24,$x00,$key_ # re-pre-load round[1]
1602 subic $len,$len,129 # $len-=129
1604 addi $len,$len,1 # $len-=128 really
1612 lvx v25,$x10,$key_ # re-pre-load round[2]
1627 lvx_u $in6,$x60,$inp
1629 lvx_u $in7,$x70,$inp
1645 le?vperm $in6,$in6,$in6,$inpperm
1647 le?vperm $in7,$in7,$in7,$inpperm
1651 # loop inX-in7 are loaded
1653 subfe. r0,r0,r0 # borrow?-1:0
1667 vxor $in6,$in6,v31
1669 vxor $in7,$in7,v31
1671 bne Lctr32_enc8x_break # did $len-129 borrow?
1688 vcipherlast $in6,$out6,$in6
1691 vcipherlast $in7,$out7,$in7
1715 le?vperm $in6,$in6,$in6,$inpperm
1718 le?vperm $in7,$in7,$in7,$inpperm
1720 stvx_u $in6,$x60,$out
1722 stvx_u $in7,$x70,$out
1729 cmpwi $len,-0x60
1733 cmpwi $len,-0x40
1737 cmpwi $len,-0x20
1751 vcipherlast $out6,$out6,$in6
1752 vcipherlast $out7,$out7,$in7
1780 vcipherlast $out5,$out5,$in6
1781 vcipherlast $out6,$out6,$in7
1806 vcipherlast $out4,$out4,$in6
1807 vcipherlast $out5,$out5,$in7
1829 vcipherlast $out3,$out3,$in6
1830 vcipherlast $out4,$out4,$in7
1849 vcipherlast $out2,$out2,$in6
1850 vcipherlast $out3,$out3,$in7
1866 vcipherlast $out1,$out1,$in6
1867 vcipherlast $out2,$out2,$in7
1880 vcipherlast $out0,$out0,$in6
1881 vcipherlast $out1,$out1,$in7
1892 vcipherlast $out0,$out0,$in7
1952 .size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1978 li r3,-1
1980 bltlr-
2044 li $idx,-16
2147 vspltisb $tmp,-1
2183 .size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2187 li r3,-1
2189 bltlr-
2359 vxor $inout,$inout,$tweak # :-(
2360 vxor $inout,$inout,$tweak1 # :-)
2397 vspltisb $tmp,-1
2437 .size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2447 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
2448 # v26-v31 last 6 round keys
2455 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2482 li r0,-1
2483 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
2510 subi $rounds,$rounds,3 # -4 in total
2524 stvx v24,$x00,$key_ # off-load round[1]
2527 stvx v25,$x10,$key_ # off-load round[2]
2534 stvx v24,$x00,$key_ # off-load round[3]
2537 stvx v25,$x10,$key_ # off-load round[4]
2548 lvx v24,$x00,$key_ # pre-load round[1]
2550 lvx v25,$x10,$key_ # pre-load round[2]
2650 subic $len,$len,96 # $len-=96
2662 subfe. r0,r0,r0 # borrow?-1:0
2690 # loop inX-in5 are loaded
2717 lvx v24,$x00,$key_ # re-pre-load round[1]
2731 lvx v25,$x10,$key_ # re-pre-load round[2]
2796 beq Loop_xts_enc6x # did $len-=96 borrow?
2930 lvx v24,$x00,$key_ # re-pre-load round[1]
2933 lvx v25,$x10,$key_ # re-pre-load round[2]
2965 vspltisb $out1,-1
3102 lvx v24,$x00,$key_ # re-pre-load round[1]
3111 lvx v25,$x10,$key_ # re-pre-load round[2]
3132 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3159 li r0,-1
3160 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
3187 subi $rounds,$rounds,3 # -4 in total
3201 stvx v24,$x00,$key_ # off-load round[1]
3204 stvx v25,$x10,$key_ # off-load round[2]
3211 stvx v24,$x00,$key_ # off-load round[3]
3214 stvx v25,$x10,$key_ # off-load round[4]
3225 lvx v24,$x00,$key_ # pre-load round[1]
3227 lvx v25,$x10,$key_ # pre-load round[2]
3319 subic $len,$len,96 # $len-=96
3331 subfe. r0,r0,r0 # borrow?-1:0
3359 # loop inX-in5 are loaded
3386 lvx v24,$x00,$key_ # re-pre-load round[1]
3400 lvx v25,$x10,$key_ # re-pre-load round[2]
3463 beq Loop_xts_dec6x # did $len-=96 borrow?
3601 lvx v24,$x00,$key_ # re-pre-load round[1]
3604 lvx v25,$x10,$key_ # re-pre-load round[2]
3653 lvx v24,$x00,$key_ # re-pre-load round[1]
3656 lvx v25,$x10,$key_ # re-pre-load round[2]
3670 vspltisb $out1,-1
3807 lvx v24,$x00,$key_ # re-pre-load round[1]
3816 lvx v25,$x10,$key_ # re-pre-load round[2]
3841 # constants table endian-specific conversion
3842 if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3846 # convert to endian-agnostic format
3856 # little-endian conversion
3870 # instructions prefixed with '?' are endian-specific and need
3872 if ($flavour =~ /le$/o) { # little-endian
3877 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3878 s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3879 s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3880 } else { # big-endian
3883 s/\?([a-z]+)/$1/o;