1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * aes-ce-glue.c - wrapper code for ARMv8 AES
4 *
5 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <linux/unaligned.h>
12 #include <crypto/aes.h>
13 #include <crypto/ctr.h>
14 #include <crypto/internal/simd.h>
15 #include <crypto/internal/skcipher.h>
16 #include <crypto/scatterwalk.h>
17 #include <linux/cpufeature.h>
18 #include <linux/module.h>
19 #include <crypto/xts.h>
20
21 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
22 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
23 MODULE_LICENSE("GPL v2");
24
25 /* defined in aes-ce-core.S */
26 asmlinkage u32 ce_aes_sub(u32 input);
27 asmlinkage void ce_aes_invert(void *dst, void *src);
28
29 asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
30 int rounds, int blocks);
31 asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
32 int rounds, int blocks);
33
34 asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
35 int rounds, int blocks, u8 iv[]);
36 asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
37 int rounds, int blocks, u8 iv[]);
38 asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
39 int rounds, int bytes, u8 const iv[]);
40 asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
41 int rounds, int bytes, u8 const iv[]);
42
43 asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
44 int rounds, int blocks, u8 ctr[]);
45
46 asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
47 int rounds, int bytes, u8 iv[],
48 u32 const rk2[], int first);
49 asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
50 int rounds, int bytes, u8 iv[],
51 u32 const rk2[], int first);
52
53 struct aes_block {
54 u8 b[AES_BLOCK_SIZE];
55 };
56
num_rounds(struct crypto_aes_ctx * ctx)57 static int num_rounds(struct crypto_aes_ctx *ctx)
58 {
59 /*
60 * # of rounds specified by AES:
61 * 128 bit key 10 rounds
62 * 192 bit key 12 rounds
63 * 256 bit key 14 rounds
64 * => n byte key => 6 + (n/4) rounds
65 */
66 return 6 + ctx->key_length / 4;
67 }
68
ce_aes_expandkey(struct crypto_aes_ctx * ctx,const u8 * in_key,unsigned int key_len)69 static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
70 unsigned int key_len)
71 {
72 /*
73 * The AES key schedule round constants
74 */
75 static u8 const rcon[] = {
76 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
77 };
78
79 u32 kwords = key_len / sizeof(u32);
80 struct aes_block *key_enc, *key_dec;
81 int i, j;
82
83 if (key_len != AES_KEYSIZE_128 &&
84 key_len != AES_KEYSIZE_192 &&
85 key_len != AES_KEYSIZE_256)
86 return -EINVAL;
87
88 ctx->key_length = key_len;
89 for (i = 0; i < kwords; i++)
90 ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
91
92 kernel_neon_begin();
93 for (i = 0; i < sizeof(rcon); i++) {
94 u32 *rki = ctx->key_enc + (i * kwords);
95 u32 *rko = rki + kwords;
96
97 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
98 rko[0] = rko[0] ^ rki[0] ^ rcon[i];
99 rko[1] = rko[0] ^ rki[1];
100 rko[2] = rko[1] ^ rki[2];
101 rko[3] = rko[2] ^ rki[3];
102
103 if (key_len == AES_KEYSIZE_192) {
104 if (i >= 7)
105 break;
106 rko[4] = rko[3] ^ rki[4];
107 rko[5] = rko[4] ^ rki[5];
108 } else if (key_len == AES_KEYSIZE_256) {
109 if (i >= 6)
110 break;
111 rko[4] = ce_aes_sub(rko[3]) ^ rki[4];
112 rko[5] = rko[4] ^ rki[5];
113 rko[6] = rko[5] ^ rki[6];
114 rko[7] = rko[6] ^ rki[7];
115 }
116 }
117
118 /*
119 * Generate the decryption keys for the Equivalent Inverse Cipher.
120 * This involves reversing the order of the round keys, and applying
121 * the Inverse Mix Columns transformation on all but the first and
122 * the last one.
123 */
124 key_enc = (struct aes_block *)ctx->key_enc;
125 key_dec = (struct aes_block *)ctx->key_dec;
126 j = num_rounds(ctx);
127
128 key_dec[0] = key_enc[j];
129 for (i = 1, j--; j > 0; i++, j--)
130 ce_aes_invert(key_dec + i, key_enc + j);
131 key_dec[i] = key_enc[0];
132
133 kernel_neon_end();
134 return 0;
135 }
136
ce_aes_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)137 static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
138 unsigned int key_len)
139 {
140 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
141
142 return ce_aes_expandkey(ctx, in_key, key_len);
143 }
144
145 struct crypto_aes_xts_ctx {
146 struct crypto_aes_ctx key1;
147 struct crypto_aes_ctx __aligned(8) key2;
148 };
149
xts_set_key(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)150 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
151 unsigned int key_len)
152 {
153 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
154 int ret;
155
156 ret = xts_verify_key(tfm, in_key, key_len);
157 if (ret)
158 return ret;
159
160 ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2);
161 if (!ret)
162 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2],
163 key_len / 2);
164 return ret;
165 }
166
ecb_encrypt(struct skcipher_request * req)167 static int ecb_encrypt(struct skcipher_request *req)
168 {
169 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
170 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
171 struct skcipher_walk walk;
172 unsigned int blocks;
173 int err;
174
175 err = skcipher_walk_virt(&walk, req, false);
176
177 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
178 kernel_neon_begin();
179 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
180 ctx->key_enc, num_rounds(ctx), blocks);
181 kernel_neon_end();
182 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
183 }
184 return err;
185 }
186
ecb_decrypt(struct skcipher_request * req)187 static int ecb_decrypt(struct skcipher_request *req)
188 {
189 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
190 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
191 struct skcipher_walk walk;
192 unsigned int blocks;
193 int err;
194
195 err = skcipher_walk_virt(&walk, req, false);
196
197 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
198 kernel_neon_begin();
199 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
200 ctx->key_dec, num_rounds(ctx), blocks);
201 kernel_neon_end();
202 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
203 }
204 return err;
205 }
206
cbc_encrypt_walk(struct skcipher_request * req,struct skcipher_walk * walk)207 static int cbc_encrypt_walk(struct skcipher_request *req,
208 struct skcipher_walk *walk)
209 {
210 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
211 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
212 unsigned int blocks;
213 int err = 0;
214
215 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
216 kernel_neon_begin();
217 ce_aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
218 ctx->key_enc, num_rounds(ctx), blocks,
219 walk->iv);
220 kernel_neon_end();
221 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
222 }
223 return err;
224 }
225
cbc_encrypt(struct skcipher_request * req)226 static int cbc_encrypt(struct skcipher_request *req)
227 {
228 struct skcipher_walk walk;
229 int err;
230
231 err = skcipher_walk_virt(&walk, req, false);
232 if (err)
233 return err;
234 return cbc_encrypt_walk(req, &walk);
235 }
236
cbc_decrypt_walk(struct skcipher_request * req,struct skcipher_walk * walk)237 static int cbc_decrypt_walk(struct skcipher_request *req,
238 struct skcipher_walk *walk)
239 {
240 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
241 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
242 unsigned int blocks;
243 int err = 0;
244
245 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
246 kernel_neon_begin();
247 ce_aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
248 ctx->key_dec, num_rounds(ctx), blocks,
249 walk->iv);
250 kernel_neon_end();
251 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
252 }
253 return err;
254 }
255
cbc_decrypt(struct skcipher_request * req)256 static int cbc_decrypt(struct skcipher_request *req)
257 {
258 struct skcipher_walk walk;
259 int err;
260
261 err = skcipher_walk_virt(&walk, req, false);
262 if (err)
263 return err;
264 return cbc_decrypt_walk(req, &walk);
265 }
266
cts_cbc_encrypt(struct skcipher_request * req)267 static int cts_cbc_encrypt(struct skcipher_request *req)
268 {
269 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
270 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
271 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
272 struct scatterlist *src = req->src, *dst = req->dst;
273 struct scatterlist sg_src[2], sg_dst[2];
274 struct skcipher_request subreq;
275 struct skcipher_walk walk;
276 int err;
277
278 skcipher_request_set_tfm(&subreq, tfm);
279 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
280 NULL, NULL);
281
282 if (req->cryptlen <= AES_BLOCK_SIZE) {
283 if (req->cryptlen < AES_BLOCK_SIZE)
284 return -EINVAL;
285 cbc_blocks = 1;
286 }
287
288 if (cbc_blocks > 0) {
289 skcipher_request_set_crypt(&subreq, req->src, req->dst,
290 cbc_blocks * AES_BLOCK_SIZE,
291 req->iv);
292
293 err = skcipher_walk_virt(&walk, &subreq, false) ?:
294 cbc_encrypt_walk(&subreq, &walk);
295 if (err)
296 return err;
297
298 if (req->cryptlen == AES_BLOCK_SIZE)
299 return 0;
300
301 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
302 if (req->dst != req->src)
303 dst = scatterwalk_ffwd(sg_dst, req->dst,
304 subreq.cryptlen);
305 }
306
307 /* handle ciphertext stealing */
308 skcipher_request_set_crypt(&subreq, src, dst,
309 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
310 req->iv);
311
312 err = skcipher_walk_virt(&walk, &subreq, false);
313 if (err)
314 return err;
315
316 kernel_neon_begin();
317 ce_aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
318 ctx->key_enc, num_rounds(ctx), walk.nbytes,
319 walk.iv);
320 kernel_neon_end();
321
322 return skcipher_walk_done(&walk, 0);
323 }
324
cts_cbc_decrypt(struct skcipher_request * req)325 static int cts_cbc_decrypt(struct skcipher_request *req)
326 {
327 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
328 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
329 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
330 struct scatterlist *src = req->src, *dst = req->dst;
331 struct scatterlist sg_src[2], sg_dst[2];
332 struct skcipher_request subreq;
333 struct skcipher_walk walk;
334 int err;
335
336 skcipher_request_set_tfm(&subreq, tfm);
337 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
338 NULL, NULL);
339
340 if (req->cryptlen <= AES_BLOCK_SIZE) {
341 if (req->cryptlen < AES_BLOCK_SIZE)
342 return -EINVAL;
343 cbc_blocks = 1;
344 }
345
346 if (cbc_blocks > 0) {
347 skcipher_request_set_crypt(&subreq, req->src, req->dst,
348 cbc_blocks * AES_BLOCK_SIZE,
349 req->iv);
350
351 err = skcipher_walk_virt(&walk, &subreq, false) ?:
352 cbc_decrypt_walk(&subreq, &walk);
353 if (err)
354 return err;
355
356 if (req->cryptlen == AES_BLOCK_SIZE)
357 return 0;
358
359 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
360 if (req->dst != req->src)
361 dst = scatterwalk_ffwd(sg_dst, req->dst,
362 subreq.cryptlen);
363 }
364
365 /* handle ciphertext stealing */
366 skcipher_request_set_crypt(&subreq, src, dst,
367 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
368 req->iv);
369
370 err = skcipher_walk_virt(&walk, &subreq, false);
371 if (err)
372 return err;
373
374 kernel_neon_begin();
375 ce_aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
376 ctx->key_dec, num_rounds(ctx), walk.nbytes,
377 walk.iv);
378 kernel_neon_end();
379
380 return skcipher_walk_done(&walk, 0);
381 }
382
ctr_encrypt(struct skcipher_request * req)383 static int ctr_encrypt(struct skcipher_request *req)
384 {
385 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
386 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
387 struct skcipher_walk walk;
388 int err, blocks;
389
390 err = skcipher_walk_virt(&walk, req, false);
391
392 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
393 kernel_neon_begin();
394 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
395 ctx->key_enc, num_rounds(ctx), blocks,
396 walk.iv);
397 kernel_neon_end();
398 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
399 }
400 if (walk.nbytes) {
401 u8 __aligned(8) tail[AES_BLOCK_SIZE];
402 unsigned int nbytes = walk.nbytes;
403 u8 *tdst = walk.dst.virt.addr;
404 u8 *tsrc = walk.src.virt.addr;
405
406 /*
407 * Tell aes_ctr_encrypt() to process a tail block.
408 */
409 blocks = -1;
410
411 kernel_neon_begin();
412 ce_aes_ctr_encrypt(tail, NULL, ctx->key_enc, num_rounds(ctx),
413 blocks, walk.iv);
414 kernel_neon_end();
415 crypto_xor_cpy(tdst, tsrc, tail, nbytes);
416 err = skcipher_walk_done(&walk, 0);
417 }
418 return err;
419 }
420
ctr_encrypt_one(struct crypto_skcipher * tfm,const u8 * src,u8 * dst)421 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
422 {
423 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
424 unsigned long flags;
425
426 /*
427 * Temporarily disable interrupts to avoid races where
428 * cachelines are evicted when the CPU is interrupted
429 * to do something else.
430 */
431 local_irq_save(flags);
432 aes_encrypt(ctx, dst, src);
433 local_irq_restore(flags);
434 }
435
ctr_encrypt_sync(struct skcipher_request * req)436 static int ctr_encrypt_sync(struct skcipher_request *req)
437 {
438 if (!crypto_simd_usable())
439 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one);
440
441 return ctr_encrypt(req);
442 }
443
xts_encrypt(struct skcipher_request * req)444 static int xts_encrypt(struct skcipher_request *req)
445 {
446 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
447 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
448 int err, first, rounds = num_rounds(&ctx->key1);
449 int tail = req->cryptlen % AES_BLOCK_SIZE;
450 struct scatterlist sg_src[2], sg_dst[2];
451 struct skcipher_request subreq;
452 struct scatterlist *src, *dst;
453 struct skcipher_walk walk;
454
455 if (req->cryptlen < AES_BLOCK_SIZE)
456 return -EINVAL;
457
458 err = skcipher_walk_virt(&walk, req, false);
459
460 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
461 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
462 AES_BLOCK_SIZE) - 2;
463
464 skcipher_walk_abort(&walk);
465
466 skcipher_request_set_tfm(&subreq, tfm);
467 skcipher_request_set_callback(&subreq,
468 skcipher_request_flags(req),
469 NULL, NULL);
470 skcipher_request_set_crypt(&subreq, req->src, req->dst,
471 xts_blocks * AES_BLOCK_SIZE,
472 req->iv);
473 req = &subreq;
474 err = skcipher_walk_virt(&walk, req, false);
475 } else {
476 tail = 0;
477 }
478
479 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
480 int nbytes = walk.nbytes;
481
482 if (walk.nbytes < walk.total)
483 nbytes &= ~(AES_BLOCK_SIZE - 1);
484
485 kernel_neon_begin();
486 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
487 ctx->key1.key_enc, rounds, nbytes, walk.iv,
488 ctx->key2.key_enc, first);
489 kernel_neon_end();
490 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
491 }
492
493 if (err || likely(!tail))
494 return err;
495
496 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
497 if (req->dst != req->src)
498 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
499
500 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
501 req->iv);
502
503 err = skcipher_walk_virt(&walk, req, false);
504 if (err)
505 return err;
506
507 kernel_neon_begin();
508 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
509 ctx->key1.key_enc, rounds, walk.nbytes, walk.iv,
510 ctx->key2.key_enc, first);
511 kernel_neon_end();
512
513 return skcipher_walk_done(&walk, 0);
514 }
515
xts_decrypt(struct skcipher_request * req)516 static int xts_decrypt(struct skcipher_request *req)
517 {
518 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
519 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
520 int err, first, rounds = num_rounds(&ctx->key1);
521 int tail = req->cryptlen % AES_BLOCK_SIZE;
522 struct scatterlist sg_src[2], sg_dst[2];
523 struct skcipher_request subreq;
524 struct scatterlist *src, *dst;
525 struct skcipher_walk walk;
526
527 if (req->cryptlen < AES_BLOCK_SIZE)
528 return -EINVAL;
529
530 err = skcipher_walk_virt(&walk, req, false);
531
532 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
533 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
534 AES_BLOCK_SIZE) - 2;
535
536 skcipher_walk_abort(&walk);
537
538 skcipher_request_set_tfm(&subreq, tfm);
539 skcipher_request_set_callback(&subreq,
540 skcipher_request_flags(req),
541 NULL, NULL);
542 skcipher_request_set_crypt(&subreq, req->src, req->dst,
543 xts_blocks * AES_BLOCK_SIZE,
544 req->iv);
545 req = &subreq;
546 err = skcipher_walk_virt(&walk, req, false);
547 } else {
548 tail = 0;
549 }
550
551 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
552 int nbytes = walk.nbytes;
553
554 if (walk.nbytes < walk.total)
555 nbytes &= ~(AES_BLOCK_SIZE - 1);
556
557 kernel_neon_begin();
558 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
559 ctx->key1.key_dec, rounds, nbytes, walk.iv,
560 ctx->key2.key_enc, first);
561 kernel_neon_end();
562 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
563 }
564
565 if (err || likely(!tail))
566 return err;
567
568 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
569 if (req->dst != req->src)
570 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
571
572 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
573 req->iv);
574
575 err = skcipher_walk_virt(&walk, req, false);
576 if (err)
577 return err;
578
579 kernel_neon_begin();
580 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
581 ctx->key1.key_dec, rounds, walk.nbytes, walk.iv,
582 ctx->key2.key_enc, first);
583 kernel_neon_end();
584
585 return skcipher_walk_done(&walk, 0);
586 }
587
588 static struct skcipher_alg aes_algs[] = { {
589 .base.cra_name = "__ecb(aes)",
590 .base.cra_driver_name = "__ecb-aes-ce",
591 .base.cra_priority = 300,
592 .base.cra_flags = CRYPTO_ALG_INTERNAL,
593 .base.cra_blocksize = AES_BLOCK_SIZE,
594 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
595 .base.cra_module = THIS_MODULE,
596
597 .min_keysize = AES_MIN_KEY_SIZE,
598 .max_keysize = AES_MAX_KEY_SIZE,
599 .setkey = ce_aes_setkey,
600 .encrypt = ecb_encrypt,
601 .decrypt = ecb_decrypt,
602 }, {
603 .base.cra_name = "__cbc(aes)",
604 .base.cra_driver_name = "__cbc-aes-ce",
605 .base.cra_priority = 300,
606 .base.cra_flags = CRYPTO_ALG_INTERNAL,
607 .base.cra_blocksize = AES_BLOCK_SIZE,
608 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
609 .base.cra_module = THIS_MODULE,
610
611 .min_keysize = AES_MIN_KEY_SIZE,
612 .max_keysize = AES_MAX_KEY_SIZE,
613 .ivsize = AES_BLOCK_SIZE,
614 .setkey = ce_aes_setkey,
615 .encrypt = cbc_encrypt,
616 .decrypt = cbc_decrypt,
617 }, {
618 .base.cra_name = "__cts(cbc(aes))",
619 .base.cra_driver_name = "__cts-cbc-aes-ce",
620 .base.cra_priority = 300,
621 .base.cra_flags = CRYPTO_ALG_INTERNAL,
622 .base.cra_blocksize = AES_BLOCK_SIZE,
623 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
624 .base.cra_module = THIS_MODULE,
625
626 .min_keysize = AES_MIN_KEY_SIZE,
627 .max_keysize = AES_MAX_KEY_SIZE,
628 .ivsize = AES_BLOCK_SIZE,
629 .walksize = 2 * AES_BLOCK_SIZE,
630 .setkey = ce_aes_setkey,
631 .encrypt = cts_cbc_encrypt,
632 .decrypt = cts_cbc_decrypt,
633 }, {
634 .base.cra_name = "__ctr(aes)",
635 .base.cra_driver_name = "__ctr-aes-ce",
636 .base.cra_priority = 300,
637 .base.cra_flags = CRYPTO_ALG_INTERNAL,
638 .base.cra_blocksize = 1,
639 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
640 .base.cra_module = THIS_MODULE,
641
642 .min_keysize = AES_MIN_KEY_SIZE,
643 .max_keysize = AES_MAX_KEY_SIZE,
644 .ivsize = AES_BLOCK_SIZE,
645 .chunksize = AES_BLOCK_SIZE,
646 .setkey = ce_aes_setkey,
647 .encrypt = ctr_encrypt,
648 .decrypt = ctr_encrypt,
649 }, {
650 .base.cra_name = "ctr(aes)",
651 .base.cra_driver_name = "ctr-aes-ce-sync",
652 .base.cra_priority = 300 - 1,
653 .base.cra_blocksize = 1,
654 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
655 .base.cra_module = THIS_MODULE,
656
657 .min_keysize = AES_MIN_KEY_SIZE,
658 .max_keysize = AES_MAX_KEY_SIZE,
659 .ivsize = AES_BLOCK_SIZE,
660 .chunksize = AES_BLOCK_SIZE,
661 .setkey = ce_aes_setkey,
662 .encrypt = ctr_encrypt_sync,
663 .decrypt = ctr_encrypt_sync,
664 }, {
665 .base.cra_name = "__xts(aes)",
666 .base.cra_driver_name = "__xts-aes-ce",
667 .base.cra_priority = 300,
668 .base.cra_flags = CRYPTO_ALG_INTERNAL,
669 .base.cra_blocksize = AES_BLOCK_SIZE,
670 .base.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
671 .base.cra_module = THIS_MODULE,
672
673 .min_keysize = 2 * AES_MIN_KEY_SIZE,
674 .max_keysize = 2 * AES_MAX_KEY_SIZE,
675 .ivsize = AES_BLOCK_SIZE,
676 .walksize = 2 * AES_BLOCK_SIZE,
677 .setkey = xts_set_key,
678 .encrypt = xts_encrypt,
679 .decrypt = xts_decrypt,
680 } };
681
682 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
683
aes_exit(void)684 static void aes_exit(void)
685 {
686 int i;
687
688 for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
689 simd_skcipher_free(aes_simd_algs[i]);
690
691 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
692 }
693
aes_init(void)694 static int __init aes_init(void)
695 {
696 struct simd_skcipher_alg *simd;
697 const char *basename;
698 const char *algname;
699 const char *drvname;
700 int err;
701 int i;
702
703 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
704 if (err)
705 return err;
706
707 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
708 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
709 continue;
710
711 algname = aes_algs[i].base.cra_name + 2;
712 drvname = aes_algs[i].base.cra_driver_name + 2;
713 basename = aes_algs[i].base.cra_driver_name;
714 simd = simd_skcipher_create_compat(aes_algs + i, algname, drvname, basename);
715 err = PTR_ERR(simd);
716 if (IS_ERR(simd))
717 goto unregister_simds;
718
719 aes_simd_algs[i] = simd;
720 }
721
722 return 0;
723
724 unregister_simds:
725 aes_exit();
726 return err;
727 }
728
729 module_cpu_feature_match(AES, aes_init);
730 module_exit(aes_exit);
731