riscv/crypto/aes-riscv64-zvkned-zvbb-zvkg.S

1 /* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
3 // This file is dual-licensed, meaning that you can use it under your
9 // a copy in the file LICENSE in the source distribution or at
12 // or
18 // Redistribution and use in source and binary forms, with or without
25 //    documentation and/or other materials provided with the distribution.
28 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 // The generated code of this file depends on the following RISC-V extensions:
40 // - RV64I
41 // - RISC-V Vector ('V') with VLEN >= 128 && VLEN < 2048
42 // - RISC-V Vector AES block cipher extension ('Zvkned')
43 // - RISC-V Vector Bit-manipulation extension ('Zvbb')
44 // - RISC-V Vector GCM/GMAC extension ('Zvkg')
51 #include "aes-macros.S"
53 #define KEYP		a0
54 #define INP		a1
55 #define OUTP		a2
56 #define LEN		a3  macro
57 #define TWEAKP		a4
59 #define LEN32		a5
60 #define TAIL_LEN	a6
61 #define VL		a7
62 #define VLMAX		t4
64 // v1-v15 contain the AES round keys, but they are used for temporaries before
66 #define TWEAKS		v16	// LMUL=4 (most of the time)
67 #define TWEAKS_BREV	v20	// LMUL=4 (most of the time)
68 #define MULTS_BREV	v24	// LMUL=4 (most of the time)
69 #define TMP0		v28
70 #define TMP1		v29
71 #define TMP2		v30
72 #define TMP3		v31
76 //	TWEAKS: N 128-bit tweaks T*(x^i) for i in 0..(N - 1)
77 //	TWEAKS_BREV: same as TWEAKS, but bit-reversed
78 //	MULTS_BREV: N 128-bit values x^N, bit-reversed.  Only if N > 1.
88 // the inputs and outputs to vgmul must be bit-reversed (we do it with vbrev8).
95 	// If there's only one block (or no blocks at all), then skip the tweak
98 	ble		LEN, t0, .Linit_single_block\@
100 	// Save a copy of T bit-reversed in v12.
104 	// Generate x^i for i in 0..(N - 1), i.e. 128-bit values 1 << i assuming
105 	// that N <= 128.  Though, this code actually requires N < 64 (or
106 	// equivalently VLEN < 2048) due to the use of 64-bit intermediate
111 	// Generate two sequences, each with N 32-bit values:
116 	// Use vzext to zero-extend the sequences to 64 bits.  Reinterpret them
117 	// as two sequences, each with 2*N 32-bit values:
125 	// widening to 64 bits per element.  When reinterpreted as N 128-bit
126 	// values, this is the needed sequence of 128-bit values 1 << i (x^i).
129 	// Copy the bit-reversed T to all N elements of TWEAKS_BREV, then
130 	// multiply by x^i.  This gives the sequence T*(x^i), bit-reversed.
140 	// Generate N copies of x^N, i.e. 128-bit values 1 << N, bit-reversed.
159 // Set the first 128 bits of MULTS_BREV to 0x40, i.e. 'x' bit-reversed.  This is
170 	// With 16 < len <= 31, there's no main loop, just ciphertext stealing.
177 	// Encrypt or decrypt VL/4 blocks.
194 	// sequence by x^N.  Store the result in both bit-reversed order and
201 	// This conflicts slightly with the behavior allowed by RISC-V Vector
214 	addi		t0, VL, -4
251 	// Note that to support in-place encryption/decryption, the load from
273 	andi		TAIL_LEN, LEN, 15
280 	sub		LEN, LEN, TAIL_LEN
281 	addi		LEN, LEN, -16
283 	srli		LEN32, LEN, 2
284 	// LEN and LEN32 now contain the total length of the blocks that will be
285 	// processed in the main loop, in bytes and 32-bit words respectively.
297 //					 const u8 *in, u8 *out, size_t len,
302 // incremental computation, but |len| must always be >= 16 (AES_BLOCK_SIZE), and
303 // |len| must be a multiple of 16 except on the last call.  If |len| is a