Lines Matching +full:in0 +full:- +full:in1
2 # SPDX-License-Identifier: GPL-2.0
12 # Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
58 # The module is endian-agnostic in sense that it supports both big-
59 # and little-endian cases. Data alignment in parallelizable modes is
64 # is aligned programmatically, which in turn guarantees exception-
72 # Add XTS subroutine, 9x on little- and 12x improvement on big-endian
76 # Current large-block performance in cycles per byte processed with
77 # 128-bit key (less is better).
79 # CBC en-/decrypt CTR XTS
106 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
107 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
108 die "can't locate ppc-xlate.pl";
121 my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
140 addi $ptr,$ptr,-0x58
152 li $ptr,-1
154 beq- Lenc_key_abort # if ($inp==0) return -1;
156 beq- Lenc_key_abort # if ($out==0) return -1;
157 li $ptr,-2
159 blt- Lenc_key_abort
161 bgt- Lenc_key_abort
163 bne- Lenc_key_abort
173 lvx $in0,0,$inp
178 lvx $in1,0,$inp
184 vperm $in0,$in0,$in1,$key # align [and byte swap in LE]
190 vspltisb $outmask,-1
202 vperm $key,$in0,$in0,$mask # rotate-n-splat
203 vsldoi $tmp,$zero,$in0,12 # >>32
204 vperm $outtail,$in0,$in0,$outperm # rotate
211 vxor $in0,$in0,$tmp
213 vxor $in0,$in0,$tmp
215 vxor $in0,$in0,$tmp
217 vxor $in0,$in0,$key
222 vperm $key,$in0,$in0,$mask # rotate-n-splat
223 vsldoi $tmp,$zero,$in0,12 # >>32
224 vperm $outtail,$in0,$in0,$outperm # rotate
231 vxor $in0,$in0,$tmp
233 vxor $in0,$in0,$tmp
235 vxor $in0,$in0,$tmp
237 vxor $in0,$in0,$key
239 vperm $key,$in0,$in0,$mask # rotate-n-splat
240 vsldoi $tmp,$zero,$in0,12 # >>32
241 vperm $outtail,$in0,$in0,$outperm # rotate
248 vxor $in0,$in0,$tmp
250 vxor $in0,$in0,$tmp
252 vxor $in0,$in0,$tmp
253 vxor $in0,$in0,$key
254 vperm $outtail,$in0,$in0,$outperm # rotate
269 vperm $outtail,$in0,$in0,$outperm # rotate
274 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
280 vperm $key,$in1,$in1,$mask # roate-n-splat
281 vsldoi $tmp,$zero,$in0,12 # >>32
284 vxor $in0,$in0,$tmp
286 vxor $in0,$in0,$tmp
288 vxor $in0,$in0,$tmp
290 vsldoi $stage,$zero,$in1,8
291 vspltw $tmp,$in0,3
292 vxor $tmp,$tmp,$in1
293 vsldoi $in1,$zero,$in1,12 # >>32
295 vxor $in1,$in1,$tmp
296 vxor $in0,$in0,$key
297 vxor $in1,$in1,$key
298 vsldoi $stage,$stage,$in0,8
300 vperm $key,$in1,$in1,$mask # rotate-n-splat
301 vsldoi $tmp,$zero,$in0,12 # >>32
309 vsldoi $stage,$in0,$in1,8
310 vxor $in0,$in0,$tmp
315 vxor $in0,$in0,$tmp
317 vxor $in0,$in0,$tmp
321 vspltw $tmp,$in0,3
322 vxor $tmp,$tmp,$in1
323 vsldoi $in1,$zero,$in1,12 # >>32
325 vxor $in1,$in1,$tmp
326 vxor $in0,$in0,$key
327 vxor $in1,$in1,$key
328 vperm $outtail,$in0,$in0,$outperm # rotate
345 vperm $outtail,$in0,$in0,$outperm # rotate
350 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
354 vperm $key,$in1,$in1,$mask # rotate-n-splat
355 vsldoi $tmp,$zero,$in0,12 # >>32
356 vperm $outtail,$in1,$in1,$outperm # rotate
363 vxor $in0,$in0,$tmp
365 vxor $in0,$in0,$tmp
367 vxor $in0,$in0,$tmp
369 vxor $in0,$in0,$key
370 vperm $outtail,$in0,$in0,$outperm # rotate
378 vspltw $key,$in0,3 # just splat
379 vsldoi $tmp,$zero,$in1,12 # >>32
382 vxor $in1,$in1,$tmp
384 vxor $in1,$in1,$tmp
386 vxor $in1,$in1,$tmp
388 vxor $in1,$in1,$key
393 lvx $in1,0,$inp # redundant in aligned case
394 vsel $in1,$outhead,$in1,$outmask
395 stvx $in1,0,$inp
406 .size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
409 $STU $sp,-$FRAME($sp)
416 bne- Ldec_key_abort
439 stw r9, -16($inp)
440 stw r10,-12($inp)
441 stw r11,-8($inp)
442 stw r12,-4($inp)
452 .size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
456 {{{ # Single block en- and decrypt procedures #
509 vspltisb v2,-1
527 .size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
534 {{{ # CBC en- and decrypt procedures #
542 bltlr-
569 vspltisb $outmask,-1
584 subi $len,$len,16 # len-=16
632 subi $len,$len,16 # len-=16
672 addi $out,$out,-1
680 vspltisb $outmask,-1
703 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
705 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
706 # v26-v31 last 6 round keys
712 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
737 li r0,-1
738 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
754 subi $rounds,$rounds,3 # -4 in total
769 stvx v24,$x00,$key_ # off-load round[1]
772 stvx v25,$x10,$key_ # off-load round[2]
779 stvx v24,$x00,$key_ # off-load round[3]
782 stvx v25,$x10,$key_ # off-load round[4]
793 lvx v24,$x00,$key_ # pre-load round[1]
795 lvx v25,$x10,$key_ # pre-load round[2]
802 lvx_u $in0,$x00,$inp # load first 8 "words"
805 lvx_u $in1,$x10,$inp
808 le?vperm $in0,$in0,$in0,$inpperm
810 le?vperm $in1,$in1,$in1,$inpperm
813 vxor $out0,$in0,$rndkey0
816 vxor $out1,$in1,$rndkey0
857 subic $len,$len,128 # $len-=128
867 subfe. r0,r0,r0 # borrow?-1:0
889 # loop inX-in7 are loaded
909 lvx v24,$x00,$key_ # re-pre-load round[1]
919 lvx v25,$x10,$key_ # re-pre-load round[2]
924 vxor $in0,$in0,v31
926 vxor $in1,$in1,v31
939 vncipherlast $out1,$out1,$in0
940 lvx_u $in0,$x00,$inp # load next input block
941 vncipherlast $out2,$out2,$in1
942 lvx_u $in1,$x10,$inp
944 le?vperm $in0,$in0,$in0,$inpperm
947 le?vperm $in1,$in1,$in1,$inpperm
967 vxor $out0,$in0,$rndkey0
971 vxor $out1,$in1,$rndkey0
992 beq Loop_cbc_dec8x # did $len-=128 borrow?
1071 vxor $in1,$in1,v31
1098 vncipherlast $out2,$out2,$in1
1284 .size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1293 # This code is written as 'ctr32', based on a 32-bit counter used
1294 # upstream. The kernel does *not* use a 32-bit counter. The kernel uses
1295 # a 128-bit counter.
1303 # 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
1304 # 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
1317 bltlr-
1352 vspltisb $outmask,-1
1379 vadduqm $ivec,$ivec,$one # Kernel change for 128-bit
1383 subic. $len,$len,1 # blocks--
1409 addi $out,$out,-1
1424 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1426 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1427 # v26-v31 last 6 round keys
1434 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1459 li r0,-1
1460 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1476 subi $rounds,$rounds,3 # -4 in total
1490 stvx v24,$x00,$key_ # off-load round[1]
1493 stvx v25,$x10,$key_ # off-load round[2]
1500 stvx v24,$x00,$key_ # off-load round[3]
1503 stvx v25,$x10,$key_ # off-load round[4]
1514 lvx v24,$x00,$key_ # pre-load round[1]
1516 lvx v25,$x10,$key_ # pre-load round[2]
1523 vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit)
1570 subic r11,$len,256 # $len-256, borrow $key_
1580 subfe r0,r0,r0 # borrow?-1:0
1600 lvx v24,$x00,$key_ # re-pre-load round[1]
1602 subic $len,$len,129 # $len-=129
1604 addi $len,$len,1 # $len-=128 really
1612 lvx v25,$x10,$key_ # re-pre-load round[2]
1615 lvx_u $in0,$x00,$inp # load input
1617 lvx_u $in1,$x10,$inp
1633 le?vperm $in0,$in0,$in0,$inpperm
1635 le?vperm $in1,$in1,$in1,$inpperm
1651 # loop inX-in7 are loaded
1653 subfe. r0,r0,r0 # borrow?-1:0
1655 vxor $in0,$in0,v31 # xor with last round key
1657 vxor $in1,$in1,v31
1671 bne Lctr32_enc8x_break # did $len-129 borrow?
1673 vcipherlast $in0,$out0,$in0
1674 vcipherlast $in1,$out1,$in1
1694 le?vperm $in0,$in0,$in0,$inpperm
1697 le?vperm $in1,$in1,$in1,$inpperm
1702 stvx_u $in0,$x00,$out
1705 stvx_u $in1,$x10,$out
1729 cmpwi $len,-0x60
1733 cmpwi $len,-0x40
1737 cmpwi $len,-0x20
1745 vcipherlast $out0,$out0,$in0
1746 vcipherlast $out1,$out1,$in1
1775 vcipherlast $out0,$out0,$in1
1952 .size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1978 li r3,-1
1980 bltlr-
2044 li $idx,-16
2147 vspltisb $tmp,-1
2183 .size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2187 li r3,-1
2189 bltlr-
2359 vxor $inout,$inout,$tweak # :-(
2360 vxor $inout,$inout,$tweak1 # :-)
2397 vspltisb $tmp,-1
2437 .size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2444 my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5));
2447 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
2448 # v26-v31 last 6 round keys
2455 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2482 li r0,-1
2483 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
2510 subi $rounds,$rounds,3 # -4 in total
2524 stvx v24,$x00,$key_ # off-load round[1]
2527 stvx v25,$x10,$key_ # off-load round[2]
2534 stvx v24,$x00,$key_ # off-load round[3]
2537 stvx v25,$x10,$key_ # off-load round[4]
2548 lvx v24,$x00,$key_ # pre-load round[1]
2550 lvx v25,$x10,$key_ # pre-load round[2]
2560 vperm $in0,$inout,$inptail,$inpperm
2566 vxor $out0,$in0,$twk0
2567 xxlor 32+$in1, 0, 0
2568 vpermxor $tweak, $tweak, $tmp, $in1
2570 lvx_u $in1,$x10,$inp
2574 le?vperm $in1,$in1,$in1,$leperm
2576 vxor $out1,$in1,$twk1
2621 xxlor 32+$in0, 0, 0
2622 vpermxor $tweak, $tweak, $tmp, $in0
2650 subic $len,$len,96 # $len-=96
2651 vxor $in0,$twk0,v31 # xor with last round key
2662 subfe. r0,r0,r0 # borrow?-1:0
2666 xxlor 32+$in1, 0, 0
2667 vpermxor $tweak, $tweak, $tmp, $in1
2670 vxor $in1,$twk1,v31
2690 # loop inX-in5 are loaded
2717 lvx v24,$x00,$key_ # re-pre-load round[1]
2731 lvx v25,$x10,$key_ # re-pre-load round[2]
2747 vcipherlast $out0,$out0,$in0
2748 lvx_u $in0,$x00,$inp # load next input block
2750 vcipherlast $out1,$out1,$in1
2751 lvx_u $in1,$x10,$inp
2753 le?vperm $in0,$in0,$in0,$leperm
2757 le?vperm $in1,$in1,$in1,$leperm
2762 xxlor 10, 32+$in0, 32+$in0
2763 xxlor 32+$in0, 0, 0
2764 vpermxor $tweak, $tweak, $tmp, $in0
2765 xxlor 32+$in0, 10, 10
2777 vxor $out0,$in0,$twk0
2780 vxor $out1,$in1,$twk1
2796 beq Loop_xts_enc6x # did $len-=96 borrow?
2812 vxor $out0,$in1,$twk0
2925 lvx_u $in0,0,$inp
2930 lvx v24,$x00,$key_ # re-pre-load round[1]
2933 lvx v25,$x10,$key_ # re-pre-load round[2]
2936 le?vperm $in0,$in0,$in0,$leperm
2939 vperm $in0,$in0,$in0,$inpperm
2957 lvx_u $in0,0,$inp
2959 le?vperm $in0,$in0,$in0,$leperm
2960 vperm $in0,$in0,$in0,$inpperm
2963 vxor $in0,$in0,$twk0
2965 vspltisb $out1,-1
2967 vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember?
3086 vxor $in1,$twk1,v31
3089 lvx_u $in0,0,$inp
3102 lvx v24,$x00,$key_ # re-pre-load round[1]
3106 le?vperm $in0,$in0,$in0,$leperm
3111 lvx v25,$x10,$key_ # re-pre-load round[2]
3115 vperm $in0,$in0,$in0,$inpperm
3122 vcipherlast $out1,$out1,$in1
3132 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3159 li r0,-1
3160 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
3187 subi $rounds,$rounds,3 # -4 in total
3201 stvx v24,$x00,$key_ # off-load round[1]
3204 stvx v25,$x10,$key_ # off-load round[2]
3211 stvx v24,$x00,$key_ # off-load round[3]
3214 stvx v25,$x10,$key_ # off-load round[4]
3225 lvx v24,$x00,$key_ # pre-load round[1]
3227 lvx v25,$x10,$key_ # pre-load round[2]
3229 vperm $in0,$inout,$inptail,$inpperm
3235 vxor $out0,$in0,$twk0
3236 xxlor 32+$in1, 0, 0
3237 vpermxor $tweak, $tweak, $tmp, $in1
3239 lvx_u $in1,$x10,$inp
3243 le?vperm $in1,$in1,$in1,$leperm
3245 vxor $out1,$in1,$twk1
3290 xxlor 32+$in0, 0, 0
3291 vpermxor $tweak, $tweak, $tmp, $in0
3319 subic $len,$len,96 # $len-=96
3320 vxor $in0,$twk0,v31 # xor with last round key
3331 subfe. r0,r0,r0 # borrow?-1:0
3335 xxlor 32+$in1, 0, 0
3336 vpermxor $tweak, $tweak, $tmp, $in1
3339 vxor $in1,$twk1,v31
3359 # loop inX-in5 are loaded
3386 lvx v24,$x00,$key_ # re-pre-load round[1]
3400 lvx v25,$x10,$key_ # re-pre-load round[2]
3416 vncipherlast $out0,$out0,$in0
3417 lvx_u $in0,$x00,$inp # load next input block
3419 vncipherlast $out1,$out1,$in1
3420 lvx_u $in1,$x10,$inp
3422 le?vperm $in0,$in0,$in0,$leperm
3426 le?vperm $in1,$in1,$in1,$leperm
3431 xxlor 10, 32+$in0, 32+$in0
3432 xxlor 32+$in0, 0, 0
3433 vpermxor $tweak, $tweak, $tmp, $in0
3434 xxlor 32+$in0, 10, 10
3445 vxor $out0,$in0,$twk0
3448 vxor $out1,$in1,$twk1
3463 beq Loop_xts_dec6x # did $len-=96 borrow?
3479 vxor $out0,$in1,$twk0
3492 vxor $out0,$in0,$twk1
3519 vxor $out0,$in0,$twk5
3544 vxor $out0,$in0,$twk4
3567 vxor $out0,$in0,$twk3
3596 lvx_u $in0,0,$inp
3601 lvx v24,$x00,$key_ # re-pre-load round[1]
3604 lvx v25,$x10,$key_ # re-pre-load round[2]
3607 le?vperm $in0,$in0,$in0,$leperm
3618 vxor $out0,$in0,$twk2
3627 lvx_u $in0,0,$inp
3628 le?vperm $in0,$in0,$in0,$leperm
3629 vxor $out0,$in0,$twk1
3645 lvx_u $in0,0,$inp
3653 lvx v24,$x00,$key_ # re-pre-load round[1]
3656 lvx v25,$x10,$key_ # re-pre-load round[2]
3659 le?vperm $in0,$in0,$in0,$leperm
3662 vperm $in0,$in0,$in0,$inpperm
3670 vspltisb $out1,-1
3672 vsel $out0,$in0,$tmp,$out0
3791 vxor $in1,$twk1,v31
3794 lvx_u $in0,0,$inp
3807 lvx v24,$x00,$key_ # re-pre-load round[1]
3811 le?vperm $in0,$in0,$in0,$leperm
3816 lvx v25,$x10,$key_ # re-pre-load round[2]
3826 vncipherlast $out1,$out1,$in1
3841 # constants table endian-specific conversion
3842 if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3846 # convert to endian-agnostic format
3856 # little-endian conversion
3870 # instructions prefixed with '?' are endian-specific and need
3872 if ($flavour =~ /le$/o) { # little-endian
3877 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3878 s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3879 s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3880 } else { # big-endian
3883 s/\?([a-z]+)/$1/o;