1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <crypto/algapi.h> 7 #include <crypto/hash.h> 8 #include <crypto/md5.h> 9 #include <crypto/sm3.h> 10 #include <crypto/internal/hash.h> 11 12 #include "cc_driver.h" 13 #include "cc_request_mgr.h" 14 #include "cc_buffer_mgr.h" 15 #include "cc_hash.h" 16 #include "cc_sram_mgr.h" 17 18 #define CC_MAX_HASH_SEQ_LEN 12 19 #define CC_MAX_OPAD_KEYS_SIZE CC_MAX_HASH_BLCK_SIZE 20 #define CC_SM3_HASH_LEN_SIZE 8 21 22 struct cc_hash_handle { 23 u32 digest_len_sram_addr; /* const value in SRAM*/ 24 u32 larval_digest_sram_addr; /* const value in SRAM */ 25 struct list_head hash_list; 26 }; 27 28 static const u32 cc_digest_len_init[] = { 29 0x00000040, 0x00000000, 0x00000000, 0x00000000 }; 30 static const u32 cc_md5_init[] = { 31 SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 }; 32 static const u32 cc_sha1_init[] = { 33 SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 }; 34 static const u32 cc_sha224_init[] = { 35 SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4, 36 SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 }; 37 static const u32 cc_sha256_init[] = { 38 SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4, 39 SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 }; 40 static const u32 cc_digest_len_sha512_init[] = { 41 0x00000080, 0x00000000, 0x00000000, 0x00000000 }; 42 43 /* 44 * Due to the way the HW works, every double word in the SHA384 and SHA512 45 * larval hashes must be stored in hi/lo order 46 */ 47 #define hilo(x) upper_32_bits(x), lower_32_bits(x) 48 static const u32 cc_sha384_init[] = { 49 hilo(SHA384_H7), hilo(SHA384_H6), hilo(SHA384_H5), hilo(SHA384_H4), 50 hilo(SHA384_H3), hilo(SHA384_H2), hilo(SHA384_H1), hilo(SHA384_H0) }; 51 static const u32 cc_sha512_init[] = { 52 hilo(SHA512_H7), hilo(SHA512_H6), hilo(SHA512_H5), hilo(SHA512_H4), 53 hilo(SHA512_H3), hilo(SHA512_H2), hilo(SHA512_H1), hilo(SHA512_H0) }; 54 55 static const u32 cc_sm3_init[] = { 56 SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE, 57 SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA }; 58 59 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[], 60 unsigned int *seq_size); 61 62 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[], 63 unsigned int *seq_size); 64 65 static const void *cc_larval_digest(struct device *dev, u32 mode); 66 67 struct cc_hash_alg { 68 struct list_head entry; 69 int hash_mode; 70 int hw_mode; 71 int inter_digestsize; 72 struct cc_drvdata *drvdata; 73 struct ahash_alg ahash_alg; 74 }; 75 76 struct hash_key_req_ctx { 77 u32 keylen; 78 dma_addr_t key_dma_addr; 79 u8 *key; 80 }; 81 82 /* hash per-session context */ 83 struct cc_hash_ctx { 84 struct cc_drvdata *drvdata; 85 /* holds the origin digest; the digest after "setkey" if HMAC,* 86 * the initial digest if HASH. 87 */ 88 u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned; 89 u8 opad_tmp_keys_buff[CC_MAX_OPAD_KEYS_SIZE] ____cacheline_aligned; 90 91 dma_addr_t opad_tmp_keys_dma_addr ____cacheline_aligned; 92 dma_addr_t digest_buff_dma_addr; 93 /* use for hmac with key large then mode block size */ 94 struct hash_key_req_ctx key_params; 95 int hash_mode; 96 int hw_mode; 97 int inter_digestsize; 98 unsigned int hash_len; 99 struct completion setkey_comp; 100 bool is_hmac; 101 }; 102 103 static void cc_set_desc(struct ahash_req_ctx *areq_ctx, struct cc_hash_ctx *ctx, 104 unsigned int flow_mode, struct cc_hw_desc desc[], 105 bool is_not_last_data, unsigned int *seq_size); 106 cc_set_endianity(u32 mode,struct cc_hw_desc * desc)107 static void cc_set_endianity(u32 mode, struct cc_hw_desc *desc) 108 { 109 if (mode == DRV_HASH_MD5 || mode == DRV_HASH_SHA384 || 110 mode == DRV_HASH_SHA512) { 111 set_bytes_swap(desc, 1); 112 } else { 113 set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN); 114 } 115 } 116 cc_map_result(struct device * dev,struct ahash_req_ctx * state,unsigned int digestsize)117 static int cc_map_result(struct device *dev, struct ahash_req_ctx *state, 118 unsigned int digestsize) 119 { 120 state->digest_result_dma_addr = 121 dma_map_single(dev, state->digest_result_buff, 122 digestsize, DMA_BIDIRECTIONAL); 123 if (dma_mapping_error(dev, state->digest_result_dma_addr)) { 124 dev_err(dev, "Mapping digest result buffer %u B for DMA failed\n", 125 digestsize); 126 return -ENOMEM; 127 } 128 dev_dbg(dev, "Mapped digest result buffer %u B at va=%pK to dma=%pad\n", 129 digestsize, state->digest_result_buff, 130 &state->digest_result_dma_addr); 131 132 return 0; 133 } 134 cc_init_req(struct device * dev,struct ahash_req_ctx * state,struct cc_hash_ctx * ctx)135 static void cc_init_req(struct device *dev, struct ahash_req_ctx *state, 136 struct cc_hash_ctx *ctx) 137 { 138 bool is_hmac = ctx->is_hmac; 139 140 memset(state, 0, sizeof(*state)); 141 142 if (is_hmac) { 143 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC && 144 ctx->hw_mode != DRV_CIPHER_CMAC) { 145 dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr, 146 ctx->inter_digestsize, 147 DMA_BIDIRECTIONAL); 148 149 memcpy(state->digest_buff, ctx->digest_buff, 150 ctx->inter_digestsize); 151 if (ctx->hash_mode == DRV_HASH_SHA512 || 152 ctx->hash_mode == DRV_HASH_SHA384) 153 memcpy(state->digest_bytes_len, 154 cc_digest_len_sha512_init, 155 ctx->hash_len); 156 else 157 memcpy(state->digest_bytes_len, 158 cc_digest_len_init, 159 ctx->hash_len); 160 } 161 162 if (ctx->hash_mode != DRV_HASH_NULL) { 163 dma_sync_single_for_cpu(dev, 164 ctx->opad_tmp_keys_dma_addr, 165 ctx->inter_digestsize, 166 DMA_BIDIRECTIONAL); 167 memcpy(state->opad_digest_buff, 168 ctx->opad_tmp_keys_buff, ctx->inter_digestsize); 169 } 170 } else { /*hash*/ 171 /* Copy the initial digests if hash flow. */ 172 const void *larval = cc_larval_digest(dev, ctx->hash_mode); 173 174 memcpy(state->digest_buff, larval, ctx->inter_digestsize); 175 } 176 } 177 cc_map_req(struct device * dev,struct ahash_req_ctx * state,struct cc_hash_ctx * ctx)178 static int cc_map_req(struct device *dev, struct ahash_req_ctx *state, 179 struct cc_hash_ctx *ctx) 180 { 181 bool is_hmac = ctx->is_hmac; 182 183 state->digest_buff_dma_addr = 184 dma_map_single(dev, state->digest_buff, 185 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 186 if (dma_mapping_error(dev, state->digest_buff_dma_addr)) { 187 dev_err(dev, "Mapping digest len %d B at va=%pK for DMA failed\n", 188 ctx->inter_digestsize, state->digest_buff); 189 return -EINVAL; 190 } 191 dev_dbg(dev, "Mapped digest %d B at va=%pK to dma=%pad\n", 192 ctx->inter_digestsize, state->digest_buff, 193 &state->digest_buff_dma_addr); 194 195 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC) { 196 state->digest_bytes_len_dma_addr = 197 dma_map_single(dev, state->digest_bytes_len, 198 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); 199 if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) { 200 dev_err(dev, "Mapping digest len %u B at va=%pK for DMA failed\n", 201 HASH_MAX_LEN_SIZE, state->digest_bytes_len); 202 goto unmap_digest_buf; 203 } 204 dev_dbg(dev, "Mapped digest len %u B at va=%pK to dma=%pad\n", 205 HASH_MAX_LEN_SIZE, state->digest_bytes_len, 206 &state->digest_bytes_len_dma_addr); 207 } 208 209 if (is_hmac && ctx->hash_mode != DRV_HASH_NULL) { 210 state->opad_digest_dma_addr = 211 dma_map_single(dev, state->opad_digest_buff, 212 ctx->inter_digestsize, 213 DMA_BIDIRECTIONAL); 214 if (dma_mapping_error(dev, state->opad_digest_dma_addr)) { 215 dev_err(dev, "Mapping opad digest %d B at va=%pK for DMA failed\n", 216 ctx->inter_digestsize, 217 state->opad_digest_buff); 218 goto unmap_digest_len; 219 } 220 dev_dbg(dev, "Mapped opad digest %d B at va=%pK to dma=%pad\n", 221 ctx->inter_digestsize, state->opad_digest_buff, 222 &state->opad_digest_dma_addr); 223 } 224 225 return 0; 226 227 unmap_digest_len: 228 if (state->digest_bytes_len_dma_addr) { 229 dma_unmap_single(dev, state->digest_bytes_len_dma_addr, 230 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); 231 state->digest_bytes_len_dma_addr = 0; 232 } 233 unmap_digest_buf: 234 if (state->digest_buff_dma_addr) { 235 dma_unmap_single(dev, state->digest_buff_dma_addr, 236 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 237 state->digest_buff_dma_addr = 0; 238 } 239 240 return -EINVAL; 241 } 242 cc_unmap_req(struct device * dev,struct ahash_req_ctx * state,struct cc_hash_ctx * ctx)243 static void cc_unmap_req(struct device *dev, struct ahash_req_ctx *state, 244 struct cc_hash_ctx *ctx) 245 { 246 if (state->digest_buff_dma_addr) { 247 dma_unmap_single(dev, state->digest_buff_dma_addr, 248 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 249 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n", 250 &state->digest_buff_dma_addr); 251 state->digest_buff_dma_addr = 0; 252 } 253 if (state->digest_bytes_len_dma_addr) { 254 dma_unmap_single(dev, state->digest_bytes_len_dma_addr, 255 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); 256 dev_dbg(dev, "Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n", 257 &state->digest_bytes_len_dma_addr); 258 state->digest_bytes_len_dma_addr = 0; 259 } 260 if (state->opad_digest_dma_addr) { 261 dma_unmap_single(dev, state->opad_digest_dma_addr, 262 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 263 dev_dbg(dev, "Unmapped opad-digest: opad_digest_dma_addr=%pad\n", 264 &state->opad_digest_dma_addr); 265 state->opad_digest_dma_addr = 0; 266 } 267 } 268 cc_unmap_result(struct device * dev,struct ahash_req_ctx * state,unsigned int digestsize,u8 * result)269 static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state, 270 unsigned int digestsize, u8 *result) 271 { 272 if (state->digest_result_dma_addr) { 273 dma_unmap_single(dev, state->digest_result_dma_addr, digestsize, 274 DMA_BIDIRECTIONAL); 275 dev_dbg(dev, "unmpa digest result buffer va (%pK) pa (%pad) len %u\n", 276 state->digest_result_buff, 277 &state->digest_result_dma_addr, digestsize); 278 memcpy(result, state->digest_result_buff, digestsize); 279 } 280 state->digest_result_dma_addr = 0; 281 } 282 cc_update_complete(struct device * dev,void * cc_req,int err)283 static void cc_update_complete(struct device *dev, void *cc_req, int err) 284 { 285 struct ahash_request *req = (struct ahash_request *)cc_req; 286 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 287 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 288 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 289 290 dev_dbg(dev, "req=%pK\n", req); 291 292 if (err != -EINPROGRESS) { 293 /* Not a BACKLOG notification */ 294 cc_unmap_hash_request(dev, state, req->src, false); 295 cc_unmap_req(dev, state, ctx); 296 } 297 298 ahash_request_complete(req, err); 299 } 300 cc_digest_complete(struct device * dev,void * cc_req,int err)301 static void cc_digest_complete(struct device *dev, void *cc_req, int err) 302 { 303 struct ahash_request *req = (struct ahash_request *)cc_req; 304 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 305 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 306 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 307 u32 digestsize = crypto_ahash_digestsize(tfm); 308 309 dev_dbg(dev, "req=%pK\n", req); 310 311 if (err != -EINPROGRESS) { 312 /* Not a BACKLOG notification */ 313 cc_unmap_hash_request(dev, state, req->src, false); 314 cc_unmap_result(dev, state, digestsize, req->result); 315 cc_unmap_req(dev, state, ctx); 316 } 317 318 ahash_request_complete(req, err); 319 } 320 cc_hash_complete(struct device * dev,void * cc_req,int err)321 static void cc_hash_complete(struct device *dev, void *cc_req, int err) 322 { 323 struct ahash_request *req = (struct ahash_request *)cc_req; 324 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 325 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 326 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 327 u32 digestsize = crypto_ahash_digestsize(tfm); 328 329 dev_dbg(dev, "req=%pK\n", req); 330 331 if (err != -EINPROGRESS) { 332 /* Not a BACKLOG notification */ 333 cc_unmap_hash_request(dev, state, req->src, false); 334 cc_unmap_result(dev, state, digestsize, req->result); 335 cc_unmap_req(dev, state, ctx); 336 } 337 338 ahash_request_complete(req, err); 339 } 340 cc_fin_result(struct cc_hw_desc * desc,struct ahash_request * req,int idx)341 static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req, 342 int idx) 343 { 344 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 346 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 347 u32 digestsize = crypto_ahash_digestsize(tfm); 348 349 /* Get final MAC result */ 350 hw_desc_init(&desc[idx]); 351 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 352 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, digestsize, 353 NS_BIT, 1); 354 set_queue_last_ind(ctx->drvdata, &desc[idx]); 355 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 356 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 357 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 358 cc_set_endianity(ctx->hash_mode, &desc[idx]); 359 idx++; 360 361 return idx; 362 } 363 cc_fin_hmac(struct cc_hw_desc * desc,struct ahash_request * req,int idx)364 static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req, 365 int idx) 366 { 367 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 368 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 369 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 370 u32 digestsize = crypto_ahash_digestsize(tfm); 371 372 /* store the hash digest result in the context */ 373 hw_desc_init(&desc[idx]); 374 set_cipher_mode(&desc[idx], ctx->hw_mode); 375 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, digestsize, 376 NS_BIT, 0); 377 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 378 cc_set_endianity(ctx->hash_mode, &desc[idx]); 379 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 380 idx++; 381 382 /* Loading hash opad xor key state */ 383 hw_desc_init(&desc[idx]); 384 set_cipher_mode(&desc[idx], ctx->hw_mode); 385 set_din_type(&desc[idx], DMA_DLLI, state->opad_digest_dma_addr, 386 ctx->inter_digestsize, NS_BIT); 387 set_flow_mode(&desc[idx], S_DIN_to_HASH); 388 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 389 idx++; 390 391 /* Load the hash current length */ 392 hw_desc_init(&desc[idx]); 393 set_cipher_mode(&desc[idx], ctx->hw_mode); 394 set_din_sram(&desc[idx], 395 cc_digest_len_addr(ctx->drvdata, ctx->hash_mode), 396 ctx->hash_len); 397 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 398 set_flow_mode(&desc[idx], S_DIN_to_HASH); 399 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 400 idx++; 401 402 /* Memory Barrier: wait for IPAD/OPAD axi write to complete */ 403 hw_desc_init(&desc[idx]); 404 set_din_no_dma(&desc[idx], 0, 0xfffff0); 405 set_dout_no_dma(&desc[idx], 0, 0, 1); 406 idx++; 407 408 /* Perform HASH update */ 409 hw_desc_init(&desc[idx]); 410 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 411 digestsize, NS_BIT); 412 set_flow_mode(&desc[idx], DIN_HASH); 413 idx++; 414 415 return idx; 416 } 417 cc_hash_digest(struct ahash_request * req)418 static int cc_hash_digest(struct ahash_request *req) 419 { 420 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 421 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 422 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 423 u32 digestsize = crypto_ahash_digestsize(tfm); 424 struct scatterlist *src = req->src; 425 unsigned int nbytes = req->nbytes; 426 u8 *result = req->result; 427 struct device *dev = drvdata_to_dev(ctx->drvdata); 428 bool is_hmac = ctx->is_hmac; 429 struct cc_crypto_req cc_req = {}; 430 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 431 u32 larval_digest_addr; 432 int idx = 0; 433 int rc = 0; 434 gfp_t flags = cc_gfp_flags(&req->base); 435 436 dev_dbg(dev, "===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash", 437 nbytes); 438 439 cc_init_req(dev, state, ctx); 440 441 if (cc_map_req(dev, state, ctx)) { 442 dev_err(dev, "map_ahash_source() failed\n"); 443 return -ENOMEM; 444 } 445 446 if (cc_map_result(dev, state, digestsize)) { 447 dev_err(dev, "map_ahash_digest() failed\n"); 448 cc_unmap_req(dev, state, ctx); 449 return -ENOMEM; 450 } 451 452 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1, 453 flags)) { 454 dev_err(dev, "map_ahash_request_final() failed\n"); 455 cc_unmap_result(dev, state, digestsize, result); 456 cc_unmap_req(dev, state, ctx); 457 return -ENOMEM; 458 } 459 460 /* Setup request structure */ 461 cc_req.user_cb = cc_digest_complete; 462 cc_req.user_arg = req; 463 464 /* If HMAC then load hash IPAD xor key, if HASH then load initial 465 * digest 466 */ 467 hw_desc_init(&desc[idx]); 468 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 469 if (is_hmac) { 470 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 471 ctx->inter_digestsize, NS_BIT); 472 } else { 473 larval_digest_addr = cc_larval_digest_addr(ctx->drvdata, 474 ctx->hash_mode); 475 set_din_sram(&desc[idx], larval_digest_addr, 476 ctx->inter_digestsize); 477 } 478 set_flow_mode(&desc[idx], S_DIN_to_HASH); 479 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 480 idx++; 481 482 /* Load the hash current length */ 483 hw_desc_init(&desc[idx]); 484 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 485 486 if (is_hmac) { 487 set_din_type(&desc[idx], DMA_DLLI, 488 state->digest_bytes_len_dma_addr, 489 ctx->hash_len, NS_BIT); 490 } else { 491 set_din_const(&desc[idx], 0, ctx->hash_len); 492 if (nbytes) 493 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 494 else 495 set_cipher_do(&desc[idx], DO_PAD); 496 } 497 set_flow_mode(&desc[idx], S_DIN_to_HASH); 498 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 499 idx++; 500 501 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx); 502 503 if (is_hmac) { 504 /* HW last hash block padding (aka. "DO_PAD") */ 505 hw_desc_init(&desc[idx]); 506 set_cipher_mode(&desc[idx], ctx->hw_mode); 507 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 508 ctx->hash_len, NS_BIT, 0); 509 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 510 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 511 set_cipher_do(&desc[idx], DO_PAD); 512 idx++; 513 514 idx = cc_fin_hmac(desc, req, idx); 515 } 516 517 idx = cc_fin_result(desc, req, idx); 518 519 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 520 if (rc != -EINPROGRESS && rc != -EBUSY) { 521 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 522 cc_unmap_hash_request(dev, state, src, true); 523 cc_unmap_result(dev, state, digestsize, result); 524 cc_unmap_req(dev, state, ctx); 525 } 526 return rc; 527 } 528 cc_restore_hash(struct cc_hw_desc * desc,struct cc_hash_ctx * ctx,struct ahash_req_ctx * state,unsigned int idx)529 static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx, 530 struct ahash_req_ctx *state, unsigned int idx) 531 { 532 /* Restore hash digest */ 533 hw_desc_init(&desc[idx]); 534 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 535 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 536 ctx->inter_digestsize, NS_BIT); 537 set_flow_mode(&desc[idx], S_DIN_to_HASH); 538 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 539 idx++; 540 541 /* Restore hash current length */ 542 hw_desc_init(&desc[idx]); 543 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 544 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 545 set_din_type(&desc[idx], DMA_DLLI, state->digest_bytes_len_dma_addr, 546 ctx->hash_len, NS_BIT); 547 set_flow_mode(&desc[idx], S_DIN_to_HASH); 548 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 549 idx++; 550 551 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx); 552 553 return idx; 554 } 555 cc_hash_update(struct ahash_request * req)556 static int cc_hash_update(struct ahash_request *req) 557 { 558 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 559 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 560 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 561 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base); 562 struct scatterlist *src = req->src; 563 unsigned int nbytes = req->nbytes; 564 struct device *dev = drvdata_to_dev(ctx->drvdata); 565 struct cc_crypto_req cc_req = {}; 566 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 567 u32 idx = 0; 568 int rc; 569 gfp_t flags = cc_gfp_flags(&req->base); 570 571 dev_dbg(dev, "===== %s-update (%d) ====\n", ctx->is_hmac ? 572 "hmac" : "hash", nbytes); 573 574 if (nbytes == 0) { 575 /* no real updates required */ 576 return 0; 577 } 578 579 rc = cc_map_hash_request_update(ctx->drvdata, state, src, nbytes, 580 block_size, flags); 581 if (rc) { 582 if (rc == 1) { 583 dev_dbg(dev, " data size not require HW update %x\n", 584 nbytes); 585 /* No hardware updates are required */ 586 return 0; 587 } 588 dev_err(dev, "map_ahash_request_update() failed\n"); 589 return -ENOMEM; 590 } 591 592 if (cc_map_req(dev, state, ctx)) { 593 dev_err(dev, "map_ahash_source() failed\n"); 594 cc_unmap_hash_request(dev, state, src, true); 595 return -EINVAL; 596 } 597 598 /* Setup request structure */ 599 cc_req.user_cb = cc_update_complete; 600 cc_req.user_arg = req; 601 602 idx = cc_restore_hash(desc, ctx, state, idx); 603 604 /* store the hash digest result in context */ 605 hw_desc_init(&desc[idx]); 606 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 607 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 608 ctx->inter_digestsize, NS_BIT, 0); 609 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 610 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 611 idx++; 612 613 /* store current hash length in context */ 614 hw_desc_init(&desc[idx]); 615 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 616 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr, 617 ctx->hash_len, NS_BIT, 1); 618 set_queue_last_ind(ctx->drvdata, &desc[idx]); 619 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 620 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 621 idx++; 622 623 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 624 if (rc != -EINPROGRESS && rc != -EBUSY) { 625 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 626 cc_unmap_hash_request(dev, state, src, true); 627 cc_unmap_req(dev, state, ctx); 628 } 629 return rc; 630 } 631 cc_do_finup(struct ahash_request * req,bool update)632 static int cc_do_finup(struct ahash_request *req, bool update) 633 { 634 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 635 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 636 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 637 u32 digestsize = crypto_ahash_digestsize(tfm); 638 struct scatterlist *src = req->src; 639 unsigned int nbytes = req->nbytes; 640 u8 *result = req->result; 641 struct device *dev = drvdata_to_dev(ctx->drvdata); 642 bool is_hmac = ctx->is_hmac; 643 struct cc_crypto_req cc_req = {}; 644 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 645 unsigned int idx = 0; 646 int rc; 647 gfp_t flags = cc_gfp_flags(&req->base); 648 649 dev_dbg(dev, "===== %s-%s (%d) ====\n", is_hmac ? "hmac" : "hash", 650 update ? "finup" : "final", nbytes); 651 652 if (cc_map_req(dev, state, ctx)) { 653 dev_err(dev, "map_ahash_source() failed\n"); 654 return -EINVAL; 655 } 656 657 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, update, 658 flags)) { 659 dev_err(dev, "map_ahash_request_final() failed\n"); 660 cc_unmap_req(dev, state, ctx); 661 return -ENOMEM; 662 } 663 if (cc_map_result(dev, state, digestsize)) { 664 dev_err(dev, "map_ahash_digest() failed\n"); 665 cc_unmap_hash_request(dev, state, src, true); 666 cc_unmap_req(dev, state, ctx); 667 return -ENOMEM; 668 } 669 670 /* Setup request structure */ 671 cc_req.user_cb = cc_hash_complete; 672 cc_req.user_arg = req; 673 674 idx = cc_restore_hash(desc, ctx, state, idx); 675 676 /* Pad the hash */ 677 hw_desc_init(&desc[idx]); 678 set_cipher_do(&desc[idx], DO_PAD); 679 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 680 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr, 681 ctx->hash_len, NS_BIT, 0); 682 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 683 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 684 idx++; 685 686 if (is_hmac) 687 idx = cc_fin_hmac(desc, req, idx); 688 689 idx = cc_fin_result(desc, req, idx); 690 691 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 692 if (rc != -EINPROGRESS && rc != -EBUSY) { 693 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 694 cc_unmap_hash_request(dev, state, src, true); 695 cc_unmap_result(dev, state, digestsize, result); 696 cc_unmap_req(dev, state, ctx); 697 } 698 return rc; 699 } 700 cc_hash_finup(struct ahash_request * req)701 static int cc_hash_finup(struct ahash_request *req) 702 { 703 return cc_do_finup(req, true); 704 } 705 706 cc_hash_final(struct ahash_request * req)707 static int cc_hash_final(struct ahash_request *req) 708 { 709 return cc_do_finup(req, false); 710 } 711 cc_hash_init(struct ahash_request * req)712 static int cc_hash_init(struct ahash_request *req) 713 { 714 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 715 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 716 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 717 struct device *dev = drvdata_to_dev(ctx->drvdata); 718 719 dev_dbg(dev, "===== init (%d) ====\n", req->nbytes); 720 721 cc_init_req(dev, state, ctx); 722 723 return 0; 724 } 725 cc_hash_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)726 static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key, 727 unsigned int keylen) 728 { 729 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 730 struct cc_crypto_req cc_req = {}; 731 struct cc_hash_ctx *ctx = NULL; 732 int blocksize = 0; 733 int digestsize = 0; 734 int i, idx = 0, rc = 0; 735 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 736 u32 larval_addr; 737 struct device *dev; 738 739 ctx = crypto_ahash_ctx_dma(ahash); 740 dev = drvdata_to_dev(ctx->drvdata); 741 dev_dbg(dev, "start keylen: %d", keylen); 742 743 blocksize = crypto_tfm_alg_blocksize(&ahash->base); 744 digestsize = crypto_ahash_digestsize(ahash); 745 746 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode); 747 748 /* The keylen value distinguishes HASH in case keylen is ZERO bytes, 749 * any NON-ZERO value utilizes HMAC flow 750 */ 751 ctx->key_params.keylen = keylen; 752 ctx->key_params.key_dma_addr = 0; 753 ctx->is_hmac = true; 754 ctx->key_params.key = NULL; 755 756 if (keylen) { 757 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); 758 if (!ctx->key_params.key) 759 return -ENOMEM; 760 761 ctx->key_params.key_dma_addr = 762 dma_map_single(dev, ctx->key_params.key, keylen, 763 DMA_TO_DEVICE); 764 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { 765 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 766 ctx->key_params.key, keylen); 767 kfree_sensitive(ctx->key_params.key); 768 return -ENOMEM; 769 } 770 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", 771 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 772 773 if (keylen > blocksize) { 774 /* Load hash initial state */ 775 hw_desc_init(&desc[idx]); 776 set_cipher_mode(&desc[idx], ctx->hw_mode); 777 set_din_sram(&desc[idx], larval_addr, 778 ctx->inter_digestsize); 779 set_flow_mode(&desc[idx], S_DIN_to_HASH); 780 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 781 idx++; 782 783 /* Load the hash current length*/ 784 hw_desc_init(&desc[idx]); 785 set_cipher_mode(&desc[idx], ctx->hw_mode); 786 set_din_const(&desc[idx], 0, ctx->hash_len); 787 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 788 set_flow_mode(&desc[idx], S_DIN_to_HASH); 789 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 790 idx++; 791 792 hw_desc_init(&desc[idx]); 793 set_din_type(&desc[idx], DMA_DLLI, 794 ctx->key_params.key_dma_addr, keylen, 795 NS_BIT); 796 set_flow_mode(&desc[idx], DIN_HASH); 797 idx++; 798 799 /* Get hashed key */ 800 hw_desc_init(&desc[idx]); 801 set_cipher_mode(&desc[idx], ctx->hw_mode); 802 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, 803 digestsize, NS_BIT, 0); 804 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 805 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 806 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 807 cc_set_endianity(ctx->hash_mode, &desc[idx]); 808 idx++; 809 810 hw_desc_init(&desc[idx]); 811 set_din_const(&desc[idx], 0, (blocksize - digestsize)); 812 set_flow_mode(&desc[idx], BYPASS); 813 set_dout_dlli(&desc[idx], 814 (ctx->opad_tmp_keys_dma_addr + 815 digestsize), 816 (blocksize - digestsize), NS_BIT, 0); 817 idx++; 818 } else { 819 hw_desc_init(&desc[idx]); 820 set_din_type(&desc[idx], DMA_DLLI, 821 ctx->key_params.key_dma_addr, keylen, 822 NS_BIT); 823 set_flow_mode(&desc[idx], BYPASS); 824 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, 825 keylen, NS_BIT, 0); 826 idx++; 827 828 if ((blocksize - keylen)) { 829 hw_desc_init(&desc[idx]); 830 set_din_const(&desc[idx], 0, 831 (blocksize - keylen)); 832 set_flow_mode(&desc[idx], BYPASS); 833 set_dout_dlli(&desc[idx], 834 (ctx->opad_tmp_keys_dma_addr + 835 keylen), (blocksize - keylen), 836 NS_BIT, 0); 837 idx++; 838 } 839 } 840 } else { 841 hw_desc_init(&desc[idx]); 842 set_din_const(&desc[idx], 0, blocksize); 843 set_flow_mode(&desc[idx], BYPASS); 844 set_dout_dlli(&desc[idx], (ctx->opad_tmp_keys_dma_addr), 845 blocksize, NS_BIT, 0); 846 idx++; 847 } 848 849 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 850 if (rc) { 851 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 852 goto out; 853 } 854 855 /* calc derived HMAC key */ 856 for (idx = 0, i = 0; i < 2; i++) { 857 /* Load hash initial state */ 858 hw_desc_init(&desc[idx]); 859 set_cipher_mode(&desc[idx], ctx->hw_mode); 860 set_din_sram(&desc[idx], larval_addr, ctx->inter_digestsize); 861 set_flow_mode(&desc[idx], S_DIN_to_HASH); 862 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 863 idx++; 864 865 /* Load the hash current length*/ 866 hw_desc_init(&desc[idx]); 867 set_cipher_mode(&desc[idx], ctx->hw_mode); 868 set_din_const(&desc[idx], 0, ctx->hash_len); 869 set_flow_mode(&desc[idx], S_DIN_to_HASH); 870 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 871 idx++; 872 873 /* Prepare ipad key */ 874 hw_desc_init(&desc[idx]); 875 set_xor_val(&desc[idx], hmac_pad_const[i]); 876 set_cipher_mode(&desc[idx], ctx->hw_mode); 877 set_flow_mode(&desc[idx], S_DIN_to_HASH); 878 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 879 idx++; 880 881 /* Perform HASH update */ 882 hw_desc_init(&desc[idx]); 883 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr, 884 blocksize, NS_BIT); 885 set_cipher_mode(&desc[idx], ctx->hw_mode); 886 set_xor_active(&desc[idx]); 887 set_flow_mode(&desc[idx], DIN_HASH); 888 idx++; 889 890 /* Get the IPAD/OPAD xor key (Note, IPAD is the initial digest 891 * of the first HASH "update" state) 892 */ 893 hw_desc_init(&desc[idx]); 894 set_cipher_mode(&desc[idx], ctx->hw_mode); 895 if (i > 0) /* Not first iteration */ 896 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, 897 ctx->inter_digestsize, NS_BIT, 0); 898 else /* First iteration */ 899 set_dout_dlli(&desc[idx], ctx->digest_buff_dma_addr, 900 ctx->inter_digestsize, NS_BIT, 0); 901 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 902 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 903 idx++; 904 } 905 906 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 907 908 out: 909 if (ctx->key_params.key_dma_addr) { 910 dma_unmap_single(dev, ctx->key_params.key_dma_addr, 911 ctx->key_params.keylen, DMA_TO_DEVICE); 912 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", 913 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 914 } 915 916 kfree_sensitive(ctx->key_params.key); 917 918 return rc; 919 } 920 cc_xcbc_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)921 static int cc_xcbc_setkey(struct crypto_ahash *ahash, 922 const u8 *key, unsigned int keylen) 923 { 924 struct cc_crypto_req cc_req = {}; 925 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 926 struct device *dev = drvdata_to_dev(ctx->drvdata); 927 int rc = 0; 928 unsigned int idx = 0; 929 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 930 931 dev_dbg(dev, "===== setkey (%d) ====\n", keylen); 932 933 switch (keylen) { 934 case AES_KEYSIZE_128: 935 case AES_KEYSIZE_192: 936 case AES_KEYSIZE_256: 937 break; 938 default: 939 return -EINVAL; 940 } 941 942 ctx->key_params.keylen = keylen; 943 944 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); 945 if (!ctx->key_params.key) 946 return -ENOMEM; 947 948 ctx->key_params.key_dma_addr = 949 dma_map_single(dev, ctx->key_params.key, keylen, DMA_TO_DEVICE); 950 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { 951 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 952 key, keylen); 953 kfree_sensitive(ctx->key_params.key); 954 return -ENOMEM; 955 } 956 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", 957 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 958 959 ctx->is_hmac = true; 960 /* 1. Load the AES key */ 961 hw_desc_init(&desc[idx]); 962 set_din_type(&desc[idx], DMA_DLLI, ctx->key_params.key_dma_addr, 963 keylen, NS_BIT); 964 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 965 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 966 set_key_size_aes(&desc[idx], keylen); 967 set_flow_mode(&desc[idx], S_DIN_to_AES); 968 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 969 idx++; 970 971 hw_desc_init(&desc[idx]); 972 set_din_const(&desc[idx], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 973 set_flow_mode(&desc[idx], DIN_AES_DOUT); 974 set_dout_dlli(&desc[idx], 975 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET), 976 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); 977 idx++; 978 979 hw_desc_init(&desc[idx]); 980 set_din_const(&desc[idx], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 981 set_flow_mode(&desc[idx], DIN_AES_DOUT); 982 set_dout_dlli(&desc[idx], 983 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET), 984 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); 985 idx++; 986 987 hw_desc_init(&desc[idx]); 988 set_din_const(&desc[idx], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 989 set_flow_mode(&desc[idx], DIN_AES_DOUT); 990 set_dout_dlli(&desc[idx], 991 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET), 992 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); 993 idx++; 994 995 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 996 997 dma_unmap_single(dev, ctx->key_params.key_dma_addr, 998 ctx->key_params.keylen, DMA_TO_DEVICE); 999 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", 1000 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 1001 1002 kfree_sensitive(ctx->key_params.key); 1003 1004 return rc; 1005 } 1006 cc_cmac_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)1007 static int cc_cmac_setkey(struct crypto_ahash *ahash, 1008 const u8 *key, unsigned int keylen) 1009 { 1010 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 1011 struct device *dev = drvdata_to_dev(ctx->drvdata); 1012 1013 dev_dbg(dev, "===== setkey (%d) ====\n", keylen); 1014 1015 ctx->is_hmac = true; 1016 1017 switch (keylen) { 1018 case AES_KEYSIZE_128: 1019 case AES_KEYSIZE_192: 1020 case AES_KEYSIZE_256: 1021 break; 1022 default: 1023 return -EINVAL; 1024 } 1025 1026 ctx->key_params.keylen = keylen; 1027 1028 /* STAT_PHASE_1: Copy key to ctx */ 1029 1030 dma_sync_single_for_cpu(dev, ctx->opad_tmp_keys_dma_addr, 1031 keylen, DMA_TO_DEVICE); 1032 1033 memcpy(ctx->opad_tmp_keys_buff, key, keylen); 1034 if (keylen == 24) { 1035 memset(ctx->opad_tmp_keys_buff + 24, 0, 1036 CC_AES_KEY_SIZE_MAX - 24); 1037 } 1038 1039 dma_sync_single_for_device(dev, ctx->opad_tmp_keys_dma_addr, 1040 keylen, DMA_TO_DEVICE); 1041 1042 ctx->key_params.keylen = keylen; 1043 1044 return 0; 1045 } 1046 cc_free_ctx(struct cc_hash_ctx * ctx)1047 static void cc_free_ctx(struct cc_hash_ctx *ctx) 1048 { 1049 struct device *dev = drvdata_to_dev(ctx->drvdata); 1050 1051 if (ctx->digest_buff_dma_addr) { 1052 dma_unmap_single(dev, ctx->digest_buff_dma_addr, 1053 sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL); 1054 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n", 1055 &ctx->digest_buff_dma_addr); 1056 ctx->digest_buff_dma_addr = 0; 1057 } 1058 if (ctx->opad_tmp_keys_dma_addr) { 1059 dma_unmap_single(dev, ctx->opad_tmp_keys_dma_addr, 1060 sizeof(ctx->opad_tmp_keys_buff), 1061 DMA_BIDIRECTIONAL); 1062 dev_dbg(dev, "Unmapped opad-digest: opad_tmp_keys_dma_addr=%pad\n", 1063 &ctx->opad_tmp_keys_dma_addr); 1064 ctx->opad_tmp_keys_dma_addr = 0; 1065 } 1066 1067 ctx->key_params.keylen = 0; 1068 } 1069 cc_alloc_ctx(struct cc_hash_ctx * ctx)1070 static int cc_alloc_ctx(struct cc_hash_ctx *ctx) 1071 { 1072 struct device *dev = drvdata_to_dev(ctx->drvdata); 1073 1074 ctx->key_params.keylen = 0; 1075 1076 ctx->digest_buff_dma_addr = 1077 dma_map_single(dev, ctx->digest_buff, sizeof(ctx->digest_buff), 1078 DMA_BIDIRECTIONAL); 1079 if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) { 1080 dev_err(dev, "Mapping digest len %zu B at va=%pK for DMA failed\n", 1081 sizeof(ctx->digest_buff), ctx->digest_buff); 1082 goto fail; 1083 } 1084 dev_dbg(dev, "Mapped digest %zu B at va=%pK to dma=%pad\n", 1085 sizeof(ctx->digest_buff), ctx->digest_buff, 1086 &ctx->digest_buff_dma_addr); 1087 1088 ctx->opad_tmp_keys_dma_addr = 1089 dma_map_single(dev, ctx->opad_tmp_keys_buff, 1090 sizeof(ctx->opad_tmp_keys_buff), 1091 DMA_BIDIRECTIONAL); 1092 if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) { 1093 dev_err(dev, "Mapping opad digest %zu B at va=%pK for DMA failed\n", 1094 sizeof(ctx->opad_tmp_keys_buff), 1095 ctx->opad_tmp_keys_buff); 1096 goto fail; 1097 } 1098 dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n", 1099 sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff, 1100 &ctx->opad_tmp_keys_dma_addr); 1101 1102 ctx->is_hmac = false; 1103 return 0; 1104 1105 fail: 1106 cc_free_ctx(ctx); 1107 return -ENOMEM; 1108 } 1109 cc_get_hash_len(struct crypto_tfm * tfm)1110 static int cc_get_hash_len(struct crypto_tfm *tfm) 1111 { 1112 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 1113 1114 if (ctx->hash_mode == DRV_HASH_SM3) 1115 return CC_SM3_HASH_LEN_SIZE; 1116 else 1117 return cc_get_default_hash_len(ctx->drvdata); 1118 } 1119 cc_cra_init(struct crypto_tfm * tfm)1120 static int cc_cra_init(struct crypto_tfm *tfm) 1121 { 1122 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 1123 struct hash_alg_common *hash_alg_common = 1124 container_of(tfm->__crt_alg, struct hash_alg_common, base); 1125 struct ahash_alg *ahash_alg = 1126 container_of(hash_alg_common, struct ahash_alg, halg); 1127 struct cc_hash_alg *cc_alg = 1128 container_of(ahash_alg, struct cc_hash_alg, ahash_alg); 1129 1130 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), 1131 sizeof(struct ahash_req_ctx)); 1132 1133 ctx->hash_mode = cc_alg->hash_mode; 1134 ctx->hw_mode = cc_alg->hw_mode; 1135 ctx->inter_digestsize = cc_alg->inter_digestsize; 1136 ctx->drvdata = cc_alg->drvdata; 1137 ctx->hash_len = cc_get_hash_len(tfm); 1138 return cc_alloc_ctx(ctx); 1139 } 1140 cc_cra_exit(struct crypto_tfm * tfm)1141 static void cc_cra_exit(struct crypto_tfm *tfm) 1142 { 1143 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 1144 struct device *dev = drvdata_to_dev(ctx->drvdata); 1145 1146 dev_dbg(dev, "cc_cra_exit"); 1147 cc_free_ctx(ctx); 1148 } 1149 cc_mac_update(struct ahash_request * req)1150 static int cc_mac_update(struct ahash_request *req) 1151 { 1152 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1153 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1154 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1155 struct device *dev = drvdata_to_dev(ctx->drvdata); 1156 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base); 1157 struct cc_crypto_req cc_req = {}; 1158 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1159 int rc; 1160 u32 idx = 0; 1161 gfp_t flags = cc_gfp_flags(&req->base); 1162 1163 if (req->nbytes == 0) { 1164 /* no real updates required */ 1165 return 0; 1166 } 1167 1168 state->xcbc_count++; 1169 1170 rc = cc_map_hash_request_update(ctx->drvdata, state, req->src, 1171 req->nbytes, block_size, flags); 1172 if (rc) { 1173 if (rc == 1) { 1174 dev_dbg(dev, " data size not require HW update %x\n", 1175 req->nbytes); 1176 /* No hardware updates are required */ 1177 return 0; 1178 } 1179 dev_err(dev, "map_ahash_request_update() failed\n"); 1180 return -ENOMEM; 1181 } 1182 1183 if (cc_map_req(dev, state, ctx)) { 1184 dev_err(dev, "map_ahash_source() failed\n"); 1185 return -EINVAL; 1186 } 1187 1188 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) 1189 cc_setup_xcbc(req, desc, &idx); 1190 else 1191 cc_setup_cmac(req, desc, &idx); 1192 1193 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, true, &idx); 1194 1195 /* store the hash digest result in context */ 1196 hw_desc_init(&desc[idx]); 1197 set_cipher_mode(&desc[idx], ctx->hw_mode); 1198 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 1199 ctx->inter_digestsize, NS_BIT, 1); 1200 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1201 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1202 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1203 idx++; 1204 1205 /* Setup request structure */ 1206 cc_req.user_cb = cc_update_complete; 1207 cc_req.user_arg = req; 1208 1209 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1210 if (rc != -EINPROGRESS && rc != -EBUSY) { 1211 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1212 cc_unmap_hash_request(dev, state, req->src, true); 1213 cc_unmap_req(dev, state, ctx); 1214 } 1215 return rc; 1216 } 1217 cc_mac_final(struct ahash_request * req)1218 static int cc_mac_final(struct ahash_request *req) 1219 { 1220 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1221 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1222 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1223 struct device *dev = drvdata_to_dev(ctx->drvdata); 1224 struct cc_crypto_req cc_req = {}; 1225 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1226 int idx = 0; 1227 int rc = 0; 1228 u32 key_size, key_len; 1229 u32 digestsize = crypto_ahash_digestsize(tfm); 1230 gfp_t flags = cc_gfp_flags(&req->base); 1231 u32 rem_cnt = *cc_hash_buf_cnt(state); 1232 1233 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { 1234 key_size = CC_AES_128_BIT_KEY_SIZE; 1235 key_len = CC_AES_128_BIT_KEY_SIZE; 1236 } else { 1237 key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE : 1238 ctx->key_params.keylen; 1239 key_len = ctx->key_params.keylen; 1240 } 1241 1242 dev_dbg(dev, "===== final xcbc reminder (%d) ====\n", rem_cnt); 1243 1244 if (cc_map_req(dev, state, ctx)) { 1245 dev_err(dev, "map_ahash_source() failed\n"); 1246 return -EINVAL; 1247 } 1248 1249 if (cc_map_hash_request_final(ctx->drvdata, state, req->src, 1250 req->nbytes, 0, flags)) { 1251 dev_err(dev, "map_ahash_request_final() failed\n"); 1252 cc_unmap_req(dev, state, ctx); 1253 return -ENOMEM; 1254 } 1255 1256 if (cc_map_result(dev, state, digestsize)) { 1257 dev_err(dev, "map_ahash_digest() failed\n"); 1258 cc_unmap_hash_request(dev, state, req->src, true); 1259 cc_unmap_req(dev, state, ctx); 1260 return -ENOMEM; 1261 } 1262 1263 /* Setup request structure */ 1264 cc_req.user_cb = cc_hash_complete; 1265 cc_req.user_arg = req; 1266 1267 if (state->xcbc_count && rem_cnt == 0) { 1268 /* Load key for ECB decryption */ 1269 hw_desc_init(&desc[idx]); 1270 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1271 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT); 1272 set_din_type(&desc[idx], DMA_DLLI, 1273 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET), 1274 key_size, NS_BIT); 1275 set_key_size_aes(&desc[idx], key_len); 1276 set_flow_mode(&desc[idx], S_DIN_to_AES); 1277 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1278 idx++; 1279 1280 /* Initiate decryption of block state to previous 1281 * block_state-XOR-M[n] 1282 */ 1283 hw_desc_init(&desc[idx]); 1284 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 1285 CC_AES_BLOCK_SIZE, NS_BIT); 1286 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 1287 CC_AES_BLOCK_SIZE, NS_BIT, 0); 1288 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1289 idx++; 1290 1291 /* Memory Barrier: wait for axi write to complete */ 1292 hw_desc_init(&desc[idx]); 1293 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1294 set_dout_no_dma(&desc[idx], 0, 0, 1); 1295 idx++; 1296 } 1297 1298 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) 1299 cc_setup_xcbc(req, desc, &idx); 1300 else 1301 cc_setup_cmac(req, desc, &idx); 1302 1303 if (state->xcbc_count == 0) { 1304 hw_desc_init(&desc[idx]); 1305 set_cipher_mode(&desc[idx], ctx->hw_mode); 1306 set_key_size_aes(&desc[idx], key_len); 1307 set_cmac_size0_mode(&desc[idx]); 1308 set_flow_mode(&desc[idx], S_DIN_to_AES); 1309 idx++; 1310 } else if (rem_cnt > 0) { 1311 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); 1312 } else { 1313 hw_desc_init(&desc[idx]); 1314 set_din_const(&desc[idx], 0x00, CC_AES_BLOCK_SIZE); 1315 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1316 idx++; 1317 } 1318 1319 /* Get final MAC result */ 1320 hw_desc_init(&desc[idx]); 1321 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, 1322 digestsize, NS_BIT, 1); 1323 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1324 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1325 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1326 set_cipher_mode(&desc[idx], ctx->hw_mode); 1327 idx++; 1328 1329 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1330 if (rc != -EINPROGRESS && rc != -EBUSY) { 1331 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1332 cc_unmap_hash_request(dev, state, req->src, true); 1333 cc_unmap_result(dev, state, digestsize, req->result); 1334 cc_unmap_req(dev, state, ctx); 1335 } 1336 return rc; 1337 } 1338 cc_mac_finup(struct ahash_request * req)1339 static int cc_mac_finup(struct ahash_request *req) 1340 { 1341 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1342 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1343 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1344 struct device *dev = drvdata_to_dev(ctx->drvdata); 1345 struct cc_crypto_req cc_req = {}; 1346 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1347 int idx = 0; 1348 int rc = 0; 1349 u32 key_len = 0; 1350 u32 digestsize = crypto_ahash_digestsize(tfm); 1351 gfp_t flags = cc_gfp_flags(&req->base); 1352 1353 dev_dbg(dev, "===== finup xcbc(%d) ====\n", req->nbytes); 1354 if (state->xcbc_count > 0 && req->nbytes == 0) { 1355 dev_dbg(dev, "No data to update. Call to fdx_mac_final\n"); 1356 return cc_mac_final(req); 1357 } 1358 1359 if (cc_map_req(dev, state, ctx)) { 1360 dev_err(dev, "map_ahash_source() failed\n"); 1361 return -EINVAL; 1362 } 1363 1364 if (cc_map_hash_request_final(ctx->drvdata, state, req->src, 1365 req->nbytes, 1, flags)) { 1366 dev_err(dev, "map_ahash_request_final() failed\n"); 1367 cc_unmap_req(dev, state, ctx); 1368 return -ENOMEM; 1369 } 1370 if (cc_map_result(dev, state, digestsize)) { 1371 dev_err(dev, "map_ahash_digest() failed\n"); 1372 cc_unmap_hash_request(dev, state, req->src, true); 1373 cc_unmap_req(dev, state, ctx); 1374 return -ENOMEM; 1375 } 1376 1377 /* Setup request structure */ 1378 cc_req.user_cb = cc_hash_complete; 1379 cc_req.user_arg = req; 1380 1381 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { 1382 key_len = CC_AES_128_BIT_KEY_SIZE; 1383 cc_setup_xcbc(req, desc, &idx); 1384 } else { 1385 key_len = ctx->key_params.keylen; 1386 cc_setup_cmac(req, desc, &idx); 1387 } 1388 1389 if (req->nbytes == 0) { 1390 hw_desc_init(&desc[idx]); 1391 set_cipher_mode(&desc[idx], ctx->hw_mode); 1392 set_key_size_aes(&desc[idx], key_len); 1393 set_cmac_size0_mode(&desc[idx]); 1394 set_flow_mode(&desc[idx], S_DIN_to_AES); 1395 idx++; 1396 } else { 1397 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); 1398 } 1399 1400 /* Get final MAC result */ 1401 hw_desc_init(&desc[idx]); 1402 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, 1403 digestsize, NS_BIT, 1); 1404 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1405 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1406 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1407 set_cipher_mode(&desc[idx], ctx->hw_mode); 1408 idx++; 1409 1410 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1411 if (rc != -EINPROGRESS && rc != -EBUSY) { 1412 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1413 cc_unmap_hash_request(dev, state, req->src, true); 1414 cc_unmap_result(dev, state, digestsize, req->result); 1415 cc_unmap_req(dev, state, ctx); 1416 } 1417 return rc; 1418 } 1419 cc_mac_digest(struct ahash_request * req)1420 static int cc_mac_digest(struct ahash_request *req) 1421 { 1422 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1423 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1424 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1425 struct device *dev = drvdata_to_dev(ctx->drvdata); 1426 u32 digestsize = crypto_ahash_digestsize(tfm); 1427 struct cc_crypto_req cc_req = {}; 1428 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1429 u32 key_len; 1430 unsigned int idx = 0; 1431 int rc; 1432 gfp_t flags = cc_gfp_flags(&req->base); 1433 1434 dev_dbg(dev, "===== -digest mac (%d) ====\n", req->nbytes); 1435 1436 cc_init_req(dev, state, ctx); 1437 1438 if (cc_map_req(dev, state, ctx)) { 1439 dev_err(dev, "map_ahash_source() failed\n"); 1440 return -ENOMEM; 1441 } 1442 if (cc_map_result(dev, state, digestsize)) { 1443 dev_err(dev, "map_ahash_digest() failed\n"); 1444 cc_unmap_req(dev, state, ctx); 1445 return -ENOMEM; 1446 } 1447 1448 if (cc_map_hash_request_final(ctx->drvdata, state, req->src, 1449 req->nbytes, 1, flags)) { 1450 dev_err(dev, "map_ahash_request_final() failed\n"); 1451 cc_unmap_req(dev, state, ctx); 1452 return -ENOMEM; 1453 } 1454 1455 /* Setup request structure */ 1456 cc_req.user_cb = cc_digest_complete; 1457 cc_req.user_arg = req; 1458 1459 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { 1460 key_len = CC_AES_128_BIT_KEY_SIZE; 1461 cc_setup_xcbc(req, desc, &idx); 1462 } else { 1463 key_len = ctx->key_params.keylen; 1464 cc_setup_cmac(req, desc, &idx); 1465 } 1466 1467 if (req->nbytes == 0) { 1468 hw_desc_init(&desc[idx]); 1469 set_cipher_mode(&desc[idx], ctx->hw_mode); 1470 set_key_size_aes(&desc[idx], key_len); 1471 set_cmac_size0_mode(&desc[idx]); 1472 set_flow_mode(&desc[idx], S_DIN_to_AES); 1473 idx++; 1474 } else { 1475 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); 1476 } 1477 1478 /* Get final MAC result */ 1479 hw_desc_init(&desc[idx]); 1480 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, 1481 CC_AES_BLOCK_SIZE, NS_BIT, 1); 1482 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1483 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1484 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1485 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1486 set_cipher_mode(&desc[idx], ctx->hw_mode); 1487 idx++; 1488 1489 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1490 if (rc != -EINPROGRESS && rc != -EBUSY) { 1491 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1492 cc_unmap_hash_request(dev, state, req->src, true); 1493 cc_unmap_result(dev, state, digestsize, req->result); 1494 cc_unmap_req(dev, state, ctx); 1495 } 1496 return rc; 1497 } 1498 cc_hash_export(struct ahash_request * req,void * out)1499 static int cc_hash_export(struct ahash_request *req, void *out) 1500 { 1501 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1502 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 1503 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1504 u8 *curr_buff = cc_hash_buf(state); 1505 u32 curr_buff_cnt = *cc_hash_buf_cnt(state); 1506 const u32 tmp = CC_EXPORT_MAGIC; 1507 1508 memcpy(out, &tmp, sizeof(u32)); 1509 out += sizeof(u32); 1510 1511 memcpy(out, state->digest_buff, ctx->inter_digestsize); 1512 out += ctx->inter_digestsize; 1513 1514 memcpy(out, state->digest_bytes_len, ctx->hash_len); 1515 out += ctx->hash_len; 1516 1517 memcpy(out, &curr_buff_cnt, sizeof(u32)); 1518 out += sizeof(u32); 1519 1520 memcpy(out, curr_buff, curr_buff_cnt); 1521 1522 return 0; 1523 } 1524 cc_hash_import(struct ahash_request * req,const void * in)1525 static int cc_hash_import(struct ahash_request *req, const void *in) 1526 { 1527 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1528 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 1529 struct device *dev = drvdata_to_dev(ctx->drvdata); 1530 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1531 u32 tmp; 1532 1533 memcpy(&tmp, in, sizeof(u32)); 1534 if (tmp != CC_EXPORT_MAGIC) 1535 return -EINVAL; 1536 in += sizeof(u32); 1537 1538 cc_init_req(dev, state, ctx); 1539 1540 memcpy(state->digest_buff, in, ctx->inter_digestsize); 1541 in += ctx->inter_digestsize; 1542 1543 memcpy(state->digest_bytes_len, in, ctx->hash_len); 1544 in += ctx->hash_len; 1545 1546 /* Sanity check the data as much as possible */ 1547 memcpy(&tmp, in, sizeof(u32)); 1548 if (tmp > CC_MAX_HASH_BLCK_SIZE) 1549 return -EINVAL; 1550 in += sizeof(u32); 1551 1552 state->buf_cnt[0] = tmp; 1553 memcpy(state->buffers[0], in, tmp); 1554 1555 return 0; 1556 } 1557 1558 struct cc_hash_template { 1559 char name[CRYPTO_MAX_ALG_NAME]; 1560 char driver_name[CRYPTO_MAX_ALG_NAME]; 1561 char mac_name[CRYPTO_MAX_ALG_NAME]; 1562 char mac_driver_name[CRYPTO_MAX_ALG_NAME]; 1563 unsigned int blocksize; 1564 bool is_mac; 1565 bool synchronize; 1566 struct ahash_alg template_ahash; 1567 int hash_mode; 1568 int hw_mode; 1569 int inter_digestsize; 1570 struct cc_drvdata *drvdata; 1571 u32 min_hw_rev; 1572 enum cc_std_body std_body; 1573 }; 1574 1575 #define CC_STATE_SIZE(_x) \ 1576 ((_x) + HASH_MAX_LEN_SIZE + CC_MAX_HASH_BLCK_SIZE + (2 * sizeof(u32))) 1577 1578 /* hash descriptors */ 1579 static struct cc_hash_template driver_hash[] = { 1580 //Asynchronize hash template 1581 { 1582 .name = "sha1", 1583 .driver_name = "sha1-ccree", 1584 .mac_name = "hmac(sha1)", 1585 .mac_driver_name = "hmac-sha1-ccree", 1586 .blocksize = SHA1_BLOCK_SIZE, 1587 .is_mac = true, 1588 .synchronize = false, 1589 .template_ahash = { 1590 .init = cc_hash_init, 1591 .update = cc_hash_update, 1592 .final = cc_hash_final, 1593 .finup = cc_hash_finup, 1594 .digest = cc_hash_digest, 1595 .export = cc_hash_export, 1596 .import = cc_hash_import, 1597 .setkey = cc_hash_setkey, 1598 .halg = { 1599 .digestsize = SHA1_DIGEST_SIZE, 1600 .statesize = CC_STATE_SIZE(SHA1_DIGEST_SIZE), 1601 }, 1602 }, 1603 .hash_mode = DRV_HASH_SHA1, 1604 .hw_mode = DRV_HASH_HW_SHA1, 1605 .inter_digestsize = SHA1_DIGEST_SIZE, 1606 .min_hw_rev = CC_HW_REV_630, 1607 .std_body = CC_STD_NIST, 1608 }, 1609 { 1610 .name = "sha256", 1611 .driver_name = "sha256-ccree", 1612 .mac_name = "hmac(sha256)", 1613 .mac_driver_name = "hmac-sha256-ccree", 1614 .blocksize = SHA256_BLOCK_SIZE, 1615 .is_mac = true, 1616 .template_ahash = { 1617 .init = cc_hash_init, 1618 .update = cc_hash_update, 1619 .final = cc_hash_final, 1620 .finup = cc_hash_finup, 1621 .digest = cc_hash_digest, 1622 .export = cc_hash_export, 1623 .import = cc_hash_import, 1624 .setkey = cc_hash_setkey, 1625 .halg = { 1626 .digestsize = SHA256_DIGEST_SIZE, 1627 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE) 1628 }, 1629 }, 1630 .hash_mode = DRV_HASH_SHA256, 1631 .hw_mode = DRV_HASH_HW_SHA256, 1632 .inter_digestsize = SHA256_DIGEST_SIZE, 1633 .min_hw_rev = CC_HW_REV_630, 1634 .std_body = CC_STD_NIST, 1635 }, 1636 { 1637 .name = "sha224", 1638 .driver_name = "sha224-ccree", 1639 .mac_name = "hmac(sha224)", 1640 .mac_driver_name = "hmac-sha224-ccree", 1641 .blocksize = SHA224_BLOCK_SIZE, 1642 .is_mac = true, 1643 .template_ahash = { 1644 .init = cc_hash_init, 1645 .update = cc_hash_update, 1646 .final = cc_hash_final, 1647 .finup = cc_hash_finup, 1648 .digest = cc_hash_digest, 1649 .export = cc_hash_export, 1650 .import = cc_hash_import, 1651 .setkey = cc_hash_setkey, 1652 .halg = { 1653 .digestsize = SHA224_DIGEST_SIZE, 1654 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE), 1655 }, 1656 }, 1657 .hash_mode = DRV_HASH_SHA224, 1658 .hw_mode = DRV_HASH_HW_SHA256, 1659 .inter_digestsize = SHA256_DIGEST_SIZE, 1660 .min_hw_rev = CC_HW_REV_630, 1661 .std_body = CC_STD_NIST, 1662 }, 1663 { 1664 .name = "sha384", 1665 .driver_name = "sha384-ccree", 1666 .mac_name = "hmac(sha384)", 1667 .mac_driver_name = "hmac-sha384-ccree", 1668 .blocksize = SHA384_BLOCK_SIZE, 1669 .is_mac = true, 1670 .template_ahash = { 1671 .init = cc_hash_init, 1672 .update = cc_hash_update, 1673 .final = cc_hash_final, 1674 .finup = cc_hash_finup, 1675 .digest = cc_hash_digest, 1676 .export = cc_hash_export, 1677 .import = cc_hash_import, 1678 .setkey = cc_hash_setkey, 1679 .halg = { 1680 .digestsize = SHA384_DIGEST_SIZE, 1681 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE), 1682 }, 1683 }, 1684 .hash_mode = DRV_HASH_SHA384, 1685 .hw_mode = DRV_HASH_HW_SHA512, 1686 .inter_digestsize = SHA512_DIGEST_SIZE, 1687 .min_hw_rev = CC_HW_REV_712, 1688 .std_body = CC_STD_NIST, 1689 }, 1690 { 1691 .name = "sha512", 1692 .driver_name = "sha512-ccree", 1693 .mac_name = "hmac(sha512)", 1694 .mac_driver_name = "hmac-sha512-ccree", 1695 .blocksize = SHA512_BLOCK_SIZE, 1696 .is_mac = true, 1697 .template_ahash = { 1698 .init = cc_hash_init, 1699 .update = cc_hash_update, 1700 .final = cc_hash_final, 1701 .finup = cc_hash_finup, 1702 .digest = cc_hash_digest, 1703 .export = cc_hash_export, 1704 .import = cc_hash_import, 1705 .setkey = cc_hash_setkey, 1706 .halg = { 1707 .digestsize = SHA512_DIGEST_SIZE, 1708 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE), 1709 }, 1710 }, 1711 .hash_mode = DRV_HASH_SHA512, 1712 .hw_mode = DRV_HASH_HW_SHA512, 1713 .inter_digestsize = SHA512_DIGEST_SIZE, 1714 .min_hw_rev = CC_HW_REV_712, 1715 .std_body = CC_STD_NIST, 1716 }, 1717 { 1718 .name = "md5", 1719 .driver_name = "md5-ccree", 1720 .mac_name = "hmac(md5)", 1721 .mac_driver_name = "hmac-md5-ccree", 1722 .blocksize = MD5_HMAC_BLOCK_SIZE, 1723 .is_mac = true, 1724 .template_ahash = { 1725 .init = cc_hash_init, 1726 .update = cc_hash_update, 1727 .final = cc_hash_final, 1728 .finup = cc_hash_finup, 1729 .digest = cc_hash_digest, 1730 .export = cc_hash_export, 1731 .import = cc_hash_import, 1732 .setkey = cc_hash_setkey, 1733 .halg = { 1734 .digestsize = MD5_DIGEST_SIZE, 1735 .statesize = CC_STATE_SIZE(MD5_DIGEST_SIZE), 1736 }, 1737 }, 1738 .hash_mode = DRV_HASH_MD5, 1739 .hw_mode = DRV_HASH_HW_MD5, 1740 .inter_digestsize = MD5_DIGEST_SIZE, 1741 .min_hw_rev = CC_HW_REV_630, 1742 .std_body = CC_STD_NIST, 1743 }, 1744 { 1745 .name = "sm3", 1746 .driver_name = "sm3-ccree", 1747 .blocksize = SM3_BLOCK_SIZE, 1748 .is_mac = false, 1749 .template_ahash = { 1750 .init = cc_hash_init, 1751 .update = cc_hash_update, 1752 .final = cc_hash_final, 1753 .finup = cc_hash_finup, 1754 .digest = cc_hash_digest, 1755 .export = cc_hash_export, 1756 .import = cc_hash_import, 1757 .setkey = cc_hash_setkey, 1758 .halg = { 1759 .digestsize = SM3_DIGEST_SIZE, 1760 .statesize = CC_STATE_SIZE(SM3_DIGEST_SIZE), 1761 }, 1762 }, 1763 .hash_mode = DRV_HASH_SM3, 1764 .hw_mode = DRV_HASH_HW_SM3, 1765 .inter_digestsize = SM3_DIGEST_SIZE, 1766 .min_hw_rev = CC_HW_REV_713, 1767 .std_body = CC_STD_OSCCA, 1768 }, 1769 { 1770 .mac_name = "xcbc(aes)", 1771 .mac_driver_name = "xcbc-aes-ccree", 1772 .blocksize = AES_BLOCK_SIZE, 1773 .is_mac = true, 1774 .template_ahash = { 1775 .init = cc_hash_init, 1776 .update = cc_mac_update, 1777 .final = cc_mac_final, 1778 .finup = cc_mac_finup, 1779 .digest = cc_mac_digest, 1780 .setkey = cc_xcbc_setkey, 1781 .export = cc_hash_export, 1782 .import = cc_hash_import, 1783 .halg = { 1784 .digestsize = AES_BLOCK_SIZE, 1785 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE), 1786 }, 1787 }, 1788 .hash_mode = DRV_HASH_NULL, 1789 .hw_mode = DRV_CIPHER_XCBC_MAC, 1790 .inter_digestsize = AES_BLOCK_SIZE, 1791 .min_hw_rev = CC_HW_REV_630, 1792 .std_body = CC_STD_NIST, 1793 }, 1794 { 1795 .mac_name = "cmac(aes)", 1796 .mac_driver_name = "cmac-aes-ccree", 1797 .blocksize = AES_BLOCK_SIZE, 1798 .is_mac = true, 1799 .template_ahash = { 1800 .init = cc_hash_init, 1801 .update = cc_mac_update, 1802 .final = cc_mac_final, 1803 .finup = cc_mac_finup, 1804 .digest = cc_mac_digest, 1805 .setkey = cc_cmac_setkey, 1806 .export = cc_hash_export, 1807 .import = cc_hash_import, 1808 .halg = { 1809 .digestsize = AES_BLOCK_SIZE, 1810 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE), 1811 }, 1812 }, 1813 .hash_mode = DRV_HASH_NULL, 1814 .hw_mode = DRV_CIPHER_CMAC, 1815 .inter_digestsize = AES_BLOCK_SIZE, 1816 .min_hw_rev = CC_HW_REV_630, 1817 .std_body = CC_STD_NIST, 1818 }, 1819 }; 1820 cc_alloc_hash_alg(struct cc_hash_template * template,struct device * dev,bool keyed)1821 static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template, 1822 struct device *dev, bool keyed) 1823 { 1824 struct cc_hash_alg *t_crypto_alg; 1825 struct crypto_alg *alg; 1826 struct ahash_alg *halg; 1827 1828 t_crypto_alg = devm_kzalloc(dev, sizeof(*t_crypto_alg), GFP_KERNEL); 1829 if (!t_crypto_alg) 1830 return ERR_PTR(-ENOMEM); 1831 1832 t_crypto_alg->ahash_alg = template->template_ahash; 1833 halg = &t_crypto_alg->ahash_alg; 1834 alg = &halg->halg.base; 1835 1836 if (keyed) { 1837 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1838 template->mac_name); 1839 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1840 template->mac_driver_name); 1841 } else { 1842 halg->setkey = NULL; 1843 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1844 template->name); 1845 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1846 template->driver_name); 1847 } 1848 alg->cra_module = THIS_MODULE; 1849 alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding(); 1850 alg->cra_priority = CC_CRA_PRIO; 1851 alg->cra_blocksize = template->blocksize; 1852 alg->cra_alignmask = 0; 1853 alg->cra_exit = cc_cra_exit; 1854 1855 alg->cra_init = cc_cra_init; 1856 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 1857 1858 t_crypto_alg->hash_mode = template->hash_mode; 1859 t_crypto_alg->hw_mode = template->hw_mode; 1860 t_crypto_alg->inter_digestsize = template->inter_digestsize; 1861 1862 return t_crypto_alg; 1863 } 1864 cc_init_copy_sram(struct cc_drvdata * drvdata,const u32 * data,unsigned int size,u32 * sram_buff_ofs)1865 static int cc_init_copy_sram(struct cc_drvdata *drvdata, const u32 *data, 1866 unsigned int size, u32 *sram_buff_ofs) 1867 { 1868 struct cc_hw_desc larval_seq[CC_DIGEST_SIZE_MAX / sizeof(u32)]; 1869 unsigned int larval_seq_len = 0; 1870 int rc; 1871 1872 cc_set_sram_desc(data, *sram_buff_ofs, size / sizeof(*data), 1873 larval_seq, &larval_seq_len); 1874 rc = send_request_init(drvdata, larval_seq, larval_seq_len); 1875 if (rc) 1876 return rc; 1877 1878 *sram_buff_ofs += size; 1879 return 0; 1880 } 1881 cc_init_hash_sram(struct cc_drvdata * drvdata)1882 int cc_init_hash_sram(struct cc_drvdata *drvdata) 1883 { 1884 struct cc_hash_handle *hash_handle = drvdata->hash_handle; 1885 u32 sram_buff_ofs = hash_handle->digest_len_sram_addr; 1886 bool large_sha_supported = (drvdata->hw_rev >= CC_HW_REV_712); 1887 bool sm3_supported = (drvdata->hw_rev >= CC_HW_REV_713); 1888 int rc = 0; 1889 1890 /* Copy-to-sram digest-len */ 1891 rc = cc_init_copy_sram(drvdata, cc_digest_len_init, 1892 sizeof(cc_digest_len_init), &sram_buff_ofs); 1893 if (rc) 1894 goto init_digest_const_err; 1895 1896 if (large_sha_supported) { 1897 /* Copy-to-sram digest-len for sha384/512 */ 1898 rc = cc_init_copy_sram(drvdata, cc_digest_len_sha512_init, 1899 sizeof(cc_digest_len_sha512_init), 1900 &sram_buff_ofs); 1901 if (rc) 1902 goto init_digest_const_err; 1903 } 1904 1905 /* The initial digests offset */ 1906 hash_handle->larval_digest_sram_addr = sram_buff_ofs; 1907 1908 /* Copy-to-sram initial SHA* digests */ 1909 rc = cc_init_copy_sram(drvdata, cc_md5_init, sizeof(cc_md5_init), 1910 &sram_buff_ofs); 1911 if (rc) 1912 goto init_digest_const_err; 1913 1914 rc = cc_init_copy_sram(drvdata, cc_sha1_init, sizeof(cc_sha1_init), 1915 &sram_buff_ofs); 1916 if (rc) 1917 goto init_digest_const_err; 1918 1919 rc = cc_init_copy_sram(drvdata, cc_sha224_init, sizeof(cc_sha224_init), 1920 &sram_buff_ofs); 1921 if (rc) 1922 goto init_digest_const_err; 1923 1924 rc = cc_init_copy_sram(drvdata, cc_sha256_init, sizeof(cc_sha256_init), 1925 &sram_buff_ofs); 1926 if (rc) 1927 goto init_digest_const_err; 1928 1929 if (sm3_supported) { 1930 rc = cc_init_copy_sram(drvdata, cc_sm3_init, 1931 sizeof(cc_sm3_init), &sram_buff_ofs); 1932 if (rc) 1933 goto init_digest_const_err; 1934 } 1935 1936 if (large_sha_supported) { 1937 rc = cc_init_copy_sram(drvdata, cc_sha384_init, 1938 sizeof(cc_sha384_init), &sram_buff_ofs); 1939 if (rc) 1940 goto init_digest_const_err; 1941 1942 rc = cc_init_copy_sram(drvdata, cc_sha512_init, 1943 sizeof(cc_sha512_init), &sram_buff_ofs); 1944 if (rc) 1945 goto init_digest_const_err; 1946 } 1947 1948 init_digest_const_err: 1949 return rc; 1950 } 1951 cc_hash_alloc(struct cc_drvdata * drvdata)1952 int cc_hash_alloc(struct cc_drvdata *drvdata) 1953 { 1954 struct cc_hash_handle *hash_handle; 1955 u32 sram_buff; 1956 u32 sram_size_to_alloc; 1957 struct device *dev = drvdata_to_dev(drvdata); 1958 int rc = 0; 1959 int alg; 1960 1961 hash_handle = devm_kzalloc(dev, sizeof(*hash_handle), GFP_KERNEL); 1962 if (!hash_handle) 1963 return -ENOMEM; 1964 1965 INIT_LIST_HEAD(&hash_handle->hash_list); 1966 drvdata->hash_handle = hash_handle; 1967 1968 sram_size_to_alloc = sizeof(cc_digest_len_init) + 1969 sizeof(cc_md5_init) + 1970 sizeof(cc_sha1_init) + 1971 sizeof(cc_sha224_init) + 1972 sizeof(cc_sha256_init); 1973 1974 if (drvdata->hw_rev >= CC_HW_REV_713) 1975 sram_size_to_alloc += sizeof(cc_sm3_init); 1976 1977 if (drvdata->hw_rev >= CC_HW_REV_712) 1978 sram_size_to_alloc += sizeof(cc_digest_len_sha512_init) + 1979 sizeof(cc_sha384_init) + sizeof(cc_sha512_init); 1980 1981 sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc); 1982 if (sram_buff == NULL_SRAM_ADDR) { 1983 rc = -ENOMEM; 1984 goto fail; 1985 } 1986 1987 /* The initial digest-len offset */ 1988 hash_handle->digest_len_sram_addr = sram_buff; 1989 1990 /*must be set before the alg registration as it is being used there*/ 1991 rc = cc_init_hash_sram(drvdata); 1992 if (rc) { 1993 dev_err(dev, "Init digest CONST failed (rc=%d)\n", rc); 1994 goto fail; 1995 } 1996 1997 /* ahash registration */ 1998 for (alg = 0; alg < ARRAY_SIZE(driver_hash); alg++) { 1999 struct cc_hash_alg *t_alg; 2000 int hw_mode = driver_hash[alg].hw_mode; 2001 2002 /* Check that the HW revision and variants are suitable */ 2003 if ((driver_hash[alg].min_hw_rev > drvdata->hw_rev) || 2004 !(drvdata->std_bodies & driver_hash[alg].std_body)) 2005 continue; 2006 2007 if (driver_hash[alg].is_mac) { 2008 /* register hmac version */ 2009 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, true); 2010 if (IS_ERR(t_alg)) { 2011 rc = PTR_ERR(t_alg); 2012 dev_err(dev, "%s alg allocation failed\n", 2013 driver_hash[alg].driver_name); 2014 goto fail; 2015 } 2016 t_alg->drvdata = drvdata; 2017 2018 rc = crypto_register_ahash(&t_alg->ahash_alg); 2019 if (rc) { 2020 dev_err(dev, "%s alg registration failed\n", 2021 driver_hash[alg].driver_name); 2022 goto fail; 2023 } 2024 2025 list_add_tail(&t_alg->entry, &hash_handle->hash_list); 2026 } 2027 if (hw_mode == DRV_CIPHER_XCBC_MAC || 2028 hw_mode == DRV_CIPHER_CMAC) 2029 continue; 2030 2031 /* register hash version */ 2032 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, false); 2033 if (IS_ERR(t_alg)) { 2034 rc = PTR_ERR(t_alg); 2035 dev_err(dev, "%s alg allocation failed\n", 2036 driver_hash[alg].driver_name); 2037 goto fail; 2038 } 2039 t_alg->drvdata = drvdata; 2040 2041 rc = crypto_register_ahash(&t_alg->ahash_alg); 2042 if (rc) { 2043 dev_err(dev, "%s alg registration failed\n", 2044 driver_hash[alg].driver_name); 2045 goto fail; 2046 } 2047 2048 list_add_tail(&t_alg->entry, &hash_handle->hash_list); 2049 } 2050 2051 return 0; 2052 2053 fail: 2054 cc_hash_free(drvdata); 2055 return rc; 2056 } 2057 cc_hash_free(struct cc_drvdata * drvdata)2058 int cc_hash_free(struct cc_drvdata *drvdata) 2059 { 2060 struct cc_hash_alg *t_hash_alg, *hash_n; 2061 struct cc_hash_handle *hash_handle = drvdata->hash_handle; 2062 2063 list_for_each_entry_safe(t_hash_alg, hash_n, &hash_handle->hash_list, 2064 entry) { 2065 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 2066 list_del(&t_hash_alg->entry); 2067 } 2068 2069 return 0; 2070 } 2071 cc_setup_xcbc(struct ahash_request * areq,struct cc_hw_desc desc[],unsigned int * seq_size)2072 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[], 2073 unsigned int *seq_size) 2074 { 2075 unsigned int idx = *seq_size; 2076 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq); 2077 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2078 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 2079 2080 /* Setup XCBC MAC K1 */ 2081 hw_desc_init(&desc[idx]); 2082 set_din_type(&desc[idx], DMA_DLLI, (ctx->opad_tmp_keys_dma_addr + 2083 XCBC_MAC_K1_OFFSET), 2084 CC_AES_128_BIT_KEY_SIZE, NS_BIT); 2085 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 2086 set_hash_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC, ctx->hash_mode); 2087 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2088 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2089 set_flow_mode(&desc[idx], S_DIN_to_AES); 2090 idx++; 2091 2092 /* Setup XCBC MAC K2 */ 2093 hw_desc_init(&desc[idx]); 2094 set_din_type(&desc[idx], DMA_DLLI, 2095 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET), 2096 CC_AES_128_BIT_KEY_SIZE, NS_BIT); 2097 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 2098 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 2099 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2100 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2101 set_flow_mode(&desc[idx], S_DIN_to_AES); 2102 idx++; 2103 2104 /* Setup XCBC MAC K3 */ 2105 hw_desc_init(&desc[idx]); 2106 set_din_type(&desc[idx], DMA_DLLI, 2107 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET), 2108 CC_AES_128_BIT_KEY_SIZE, NS_BIT); 2109 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 2110 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 2111 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2112 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2113 set_flow_mode(&desc[idx], S_DIN_to_AES); 2114 idx++; 2115 2116 /* Loading MAC state */ 2117 hw_desc_init(&desc[idx]); 2118 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 2119 CC_AES_BLOCK_SIZE, NS_BIT); 2120 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 2121 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 2122 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2123 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2124 set_flow_mode(&desc[idx], S_DIN_to_AES); 2125 idx++; 2126 *seq_size = idx; 2127 } 2128 cc_setup_cmac(struct ahash_request * areq,struct cc_hw_desc desc[],unsigned int * seq_size)2129 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[], 2130 unsigned int *seq_size) 2131 { 2132 unsigned int idx = *seq_size; 2133 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq); 2134 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2135 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 2136 2137 /* Setup CMAC Key */ 2138 hw_desc_init(&desc[idx]); 2139 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr, 2140 ((ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE : 2141 ctx->key_params.keylen), NS_BIT); 2142 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 2143 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC); 2144 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2145 set_key_size_aes(&desc[idx], ctx->key_params.keylen); 2146 set_flow_mode(&desc[idx], S_DIN_to_AES); 2147 idx++; 2148 2149 /* Load MAC state */ 2150 hw_desc_init(&desc[idx]); 2151 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 2152 CC_AES_BLOCK_SIZE, NS_BIT); 2153 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 2154 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC); 2155 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2156 set_key_size_aes(&desc[idx], ctx->key_params.keylen); 2157 set_flow_mode(&desc[idx], S_DIN_to_AES); 2158 idx++; 2159 *seq_size = idx; 2160 } 2161 cc_set_desc(struct ahash_req_ctx * areq_ctx,struct cc_hash_ctx * ctx,unsigned int flow_mode,struct cc_hw_desc desc[],bool is_not_last_data,unsigned int * seq_size)2162 static void cc_set_desc(struct ahash_req_ctx *areq_ctx, 2163 struct cc_hash_ctx *ctx, unsigned int flow_mode, 2164 struct cc_hw_desc desc[], bool is_not_last_data, 2165 unsigned int *seq_size) 2166 { 2167 unsigned int idx = *seq_size; 2168 struct device *dev = drvdata_to_dev(ctx->drvdata); 2169 2170 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_DLLI) { 2171 hw_desc_init(&desc[idx]); 2172 set_din_type(&desc[idx], DMA_DLLI, 2173 sg_dma_address(areq_ctx->curr_sg), 2174 areq_ctx->curr_sg->length, NS_BIT); 2175 set_flow_mode(&desc[idx], flow_mode); 2176 idx++; 2177 } else { 2178 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) { 2179 dev_dbg(dev, " NULL mode\n"); 2180 /* nothing to build */ 2181 return; 2182 } 2183 /* bypass */ 2184 hw_desc_init(&desc[idx]); 2185 set_din_type(&desc[idx], DMA_DLLI, 2186 areq_ctx->mlli_params.mlli_dma_addr, 2187 areq_ctx->mlli_params.mlli_len, NS_BIT); 2188 set_dout_sram(&desc[idx], ctx->drvdata->mlli_sram_addr, 2189 areq_ctx->mlli_params.mlli_len); 2190 set_flow_mode(&desc[idx], BYPASS); 2191 idx++; 2192 /* process */ 2193 hw_desc_init(&desc[idx]); 2194 set_din_type(&desc[idx], DMA_MLLI, 2195 ctx->drvdata->mlli_sram_addr, 2196 areq_ctx->mlli_nents, NS_BIT); 2197 set_flow_mode(&desc[idx], flow_mode); 2198 idx++; 2199 } 2200 if (is_not_last_data) 2201 set_din_not_last_indication(&desc[(idx - 1)]); 2202 /* return updated desc sequence size */ 2203 *seq_size = idx; 2204 } 2205 cc_larval_digest(struct device * dev,u32 mode)2206 static const void *cc_larval_digest(struct device *dev, u32 mode) 2207 { 2208 switch (mode) { 2209 case DRV_HASH_MD5: 2210 return cc_md5_init; 2211 case DRV_HASH_SHA1: 2212 return cc_sha1_init; 2213 case DRV_HASH_SHA224: 2214 return cc_sha224_init; 2215 case DRV_HASH_SHA256: 2216 return cc_sha256_init; 2217 case DRV_HASH_SHA384: 2218 return cc_sha384_init; 2219 case DRV_HASH_SHA512: 2220 return cc_sha512_init; 2221 case DRV_HASH_SM3: 2222 return cc_sm3_init; 2223 default: 2224 dev_err(dev, "Invalid hash mode (%d)\n", mode); 2225 return cc_md5_init; 2226 } 2227 } 2228 2229 /** 2230 * cc_larval_digest_addr() - Get the address of the initial digest in SRAM 2231 * according to the given hash mode 2232 * 2233 * @drvdata: Associated device driver context 2234 * @mode: The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256 2235 * 2236 * Return: 2237 * The address of the initial digest in SRAM 2238 */ cc_larval_digest_addr(void * drvdata,u32 mode)2239 u32 cc_larval_digest_addr(void *drvdata, u32 mode) 2240 { 2241 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata; 2242 struct cc_hash_handle *hash_handle = _drvdata->hash_handle; 2243 struct device *dev = drvdata_to_dev(_drvdata); 2244 bool sm3_supported = (_drvdata->hw_rev >= CC_HW_REV_713); 2245 u32 addr; 2246 2247 switch (mode) { 2248 case DRV_HASH_NULL: 2249 break; /*Ignore*/ 2250 case DRV_HASH_MD5: 2251 return (hash_handle->larval_digest_sram_addr); 2252 case DRV_HASH_SHA1: 2253 return (hash_handle->larval_digest_sram_addr + 2254 sizeof(cc_md5_init)); 2255 case DRV_HASH_SHA224: 2256 return (hash_handle->larval_digest_sram_addr + 2257 sizeof(cc_md5_init) + 2258 sizeof(cc_sha1_init)); 2259 case DRV_HASH_SHA256: 2260 return (hash_handle->larval_digest_sram_addr + 2261 sizeof(cc_md5_init) + 2262 sizeof(cc_sha1_init) + 2263 sizeof(cc_sha224_init)); 2264 case DRV_HASH_SM3: 2265 return (hash_handle->larval_digest_sram_addr + 2266 sizeof(cc_md5_init) + 2267 sizeof(cc_sha1_init) + 2268 sizeof(cc_sha224_init) + 2269 sizeof(cc_sha256_init)); 2270 case DRV_HASH_SHA384: 2271 addr = (hash_handle->larval_digest_sram_addr + 2272 sizeof(cc_md5_init) + 2273 sizeof(cc_sha1_init) + 2274 sizeof(cc_sha224_init) + 2275 sizeof(cc_sha256_init)); 2276 if (sm3_supported) 2277 addr += sizeof(cc_sm3_init); 2278 return addr; 2279 case DRV_HASH_SHA512: 2280 addr = (hash_handle->larval_digest_sram_addr + 2281 sizeof(cc_md5_init) + 2282 sizeof(cc_sha1_init) + 2283 sizeof(cc_sha224_init) + 2284 sizeof(cc_sha256_init) + 2285 sizeof(cc_sha384_init)); 2286 if (sm3_supported) 2287 addr += sizeof(cc_sm3_init); 2288 return addr; 2289 default: 2290 dev_err(dev, "Invalid hash mode (%d)\n", mode); 2291 } 2292 2293 /*This is valid wrong value to avoid kernel crash*/ 2294 return hash_handle->larval_digest_sram_addr; 2295 } 2296 cc_digest_len_addr(void * drvdata,u32 mode)2297 u32 cc_digest_len_addr(void *drvdata, u32 mode) 2298 { 2299 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata; 2300 struct cc_hash_handle *hash_handle = _drvdata->hash_handle; 2301 u32 digest_len_addr = hash_handle->digest_len_sram_addr; 2302 2303 switch (mode) { 2304 case DRV_HASH_SHA1: 2305 case DRV_HASH_SHA224: 2306 case DRV_HASH_SHA256: 2307 case DRV_HASH_MD5: 2308 return digest_len_addr; 2309 case DRV_HASH_SHA384: 2310 case DRV_HASH_SHA512: 2311 return digest_len_addr + sizeof(cc_digest_len_init); 2312 default: 2313 return digest_len_addr; /*to avoid kernel crash*/ 2314 } 2315 } 2316