Lines Matching +full:dma +full:- +full:write
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2018-2020 Intel Corporation
8 #include <linux/dma-mapping.h>
20 #include "ocs-aes.h"
76 * This bit activates the DMA. When the DMA finishes, it resets
81 * this bit is reset by the DMA.
84 * terminated this bit is reset by the DMA.
131 * 11-bit value, but it is actually 10-bits.
137 * before the tag is written. For 128-bit mode this required delay is 28 OCS
138 * clock cycles. For 256-bit mode it is 36 OCS clock cycles.
156 * ------------ ---------
158 * 1 ... 15-L Nonce N
159 * 16-L ... 15 Counter i
161 * Flags = L' = L - 1
164 #define COUNTER_START(lprime) (16 - ((lprime) + 1))
175 * struct ocs_dma_linked_list - OCS DMA linked list entry.
179 * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
192 * bit [10] - KEY_HI_LO_SWAP
193 * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
194 * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
195 * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
196 * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
197 * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
198 * bit [4] - IV_SWAP_BYTES_IN_DWORD
199 * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
200 * bit [2] - DOUT_SWAP_BYTES_IN_DWORD
201 * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
202 * bit [0] - DOUT_SWAP_BYTES_IN_DWORD
206 iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); in aes_a_set_endianness()
212 iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_op_trigger()
219 aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_op_termination()
225 * Called when DMA is programmed to fetch the last batch of data.
226 * - For AES-CCM it is called for the last batch of Payload data and Ciphertext
228 * - For AES-GCM, it is called for the last batch of Plaintext data and
234 aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_set_last_gcx()
243 aes_active_reg = ioread32(aes_dev->base_reg + in aes_a_wait_last_gcx()
254 reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); in aes_a_dma_wait_input_buffer_occupancy()
262 * Called when DMA is programmed to fetch the last batch of Associated Data
268 aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_set_last_gcx_and_adata()
271 /* Set DMA src and dst transfer size to 0 */
274 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); in aes_a_dma_set_xfer_size_zero()
275 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); in aes_a_dma_set_xfer_size_zero()
278 /* Activate DMA for zero-byte transfer case. */
282 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active()
285 /* Activate DMA and enable src linked list */
290 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active_src_ll_en()
293 /* Activate DMA and enable dst linked list */
298 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active_dst_ll_en()
301 /* Activate DMA and enable src and dst linked lists */
307 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active_src_dst_ll_en()
313 iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); in aes_a_dma_reset_and_activate_perf_cntr()
315 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); in aes_a_dma_reset_and_activate_perf_cntr()
322 while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) in aes_a_dma_wait_and_deactivate_perf_cntr()
325 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); in aes_a_dma_wait_and_deactivate_perf_cntr()
328 /* Disable AES and DMA IRQ. */
335 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); in aes_irq_disable()
336 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); in aes_irq_disable()
339 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); in aes_irq_disable()
342 aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); in aes_irq_disable()
344 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); in aes_irq_disable()
347 aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); in aes_irq_disable()
349 isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); in aes_irq_disable()
351 iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); in aes_irq_disable()
354 /* Enable AES or DMA IRQ. IRQ is disabled once fired. */
358 /* Ensure DMA error interrupts are enabled */ in aes_irq_enable()
366 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); in aes_irq_enable()
370 * bits [31:3] - reserved in aes_irq_enable()
371 * bit [2] - EN_SKS_ERR in aes_irq_enable()
372 * bit [1] - EN_AES_COMPLETE in aes_irq_enable()
373 * bit [0] - reserved in aes_irq_enable()
375 iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); in aes_irq_enable()
380 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); in aes_irq_enable()
384 * bits [31:9] - reserved in aes_irq_enable()
385 * bit [8] - CPD_ERR_INT_EN in aes_irq_enable()
386 * bit [7] - OUTBUF_RD_ERR_INT_EN in aes_irq_enable()
387 * bit [6] - OUTBUF_WR_ERR_INT_EN in aes_irq_enable()
388 * bit [5] - INBUF_RD_ERR_INT_EN in aes_irq_enable()
389 * bit [4] - INBUF_WR_ERR_INT_EN in aes_irq_enable()
390 * bit [3] - BAD_COMP_INT_EN in aes_irq_enable()
391 * bit [2] - SAI_INT_EN in aes_irq_enable()
392 * bit [1] - DST_DONE_INT_EN in aes_irq_enable()
393 * bit [0] - SRC_DONE_INT_EN in aes_irq_enable()
403 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); in aes_irq_enable()
407 /* Enable and wait for IRQ (either from OCS AES engine or DMA) */
412 reinit_completion(&aes_dev->irq_completion); in ocs_aes_irq_enable_and_wait()
414 rc = wait_for_completion_interruptible(&aes_dev->irq_completion); in ocs_aes_irq_enable_and_wait()
418 return aes_dev->dma_err_mask ? -EIO : 0; in ocs_aes_irq_enable_and_wait()
421 /* Configure DMA to OCS, linked list mode */
425 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); in dma_to_ocs_aes_ll()
427 aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); in dma_to_ocs_aes_ll()
430 /* Configure DMA from OCS, linked list mode */
434 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); in dma_from_ocs_aes_ll()
436 aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); in dma_from_ocs_aes_ll()
444 /* Read DMA ISR status. */ in ocs_aes_irq_handler()
445 aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); in ocs_aes_irq_handler()
450 /* Save DMA error status. */ in ocs_aes_irq_handler()
451 aes_dev->dma_err_mask = aes_dma_isr & in ocs_aes_irq_handler()
461 complete(&aes_dev->irq_completion); in ocs_aes_irq_handler()
467 * ocs_aes_set_key() - Write key into OCS AES hardware.
468 * @aes_dev: The OCS AES device to write the key to.
470 * @key: The key to write.
484 /* OCS AES supports 128-bit and 256-bit keys only. */ in ocs_aes_set_key()
486 dev_err(aes_dev->dev, in ocs_aes_set_key()
487 "%d-bit keys not supported by AES cipher\n", in ocs_aes_set_key()
489 return -EINVAL; in ocs_aes_set_key()
491 /* OCS SM4 supports 128-bit keys only. */ in ocs_aes_set_key()
493 dev_err(aes_dev->dev, in ocs_aes_set_key()
494 "%d-bit keys not supported for SM4 cipher\n", in ocs_aes_set_key()
496 return -EINVAL; in ocs_aes_set_key()
500 return -EINVAL; in ocs_aes_set_key()
504 /* Write key to AES_KEY[0-7] registers */ in ocs_aes_set_key()
507 aes_dev->base_reg + AES_KEY_0_OFFSET + in ocs_aes_set_key()
511 * Write key size in ocs_aes_set_key()
512 * bits [31:1] - reserved in ocs_aes_set_key()
513 * bit [0] - AES_KEY_SIZE in ocs_aes_set_key()
514 * 0 - 128 bit key in ocs_aes_set_key()
515 * 1 - 256 bit key in ocs_aes_set_key()
518 iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); in ocs_aes_set_key()
523 /* Write AES_COMMAND */
533 * bit [14] - CIPHER_SELECT in set_ocs_aes_command()
534 * 0 - AES in set_ocs_aes_command()
535 * 1 - SM4 in set_ocs_aes_command()
536 * bits [11:8] - OCS_AES_MODE in set_ocs_aes_command()
537 * 0000 - ECB in set_ocs_aes_command()
538 * 0001 - CBC in set_ocs_aes_command()
539 * 0010 - CTR in set_ocs_aes_command()
540 * 0110 - CCM in set_ocs_aes_command()
541 * 0111 - GCM in set_ocs_aes_command()
542 * 1001 - CTS in set_ocs_aes_command()
543 * bits [7:6] - AES_INSTRUCTION in set_ocs_aes_command()
544 * 00 - ENCRYPT in set_ocs_aes_command()
545 * 01 - DECRYPT in set_ocs_aes_command()
546 * 10 - EXPAND in set_ocs_aes_command()
547 * 11 - BYPASS in set_ocs_aes_command()
548 * bits [3:2] - CTR_M_BITS in set_ocs_aes_command()
549 * 00 - No increment in set_ocs_aes_command()
550 * 01 - Least significant 32 bits are incremented in set_ocs_aes_command()
551 * 10 - Least significant 64 bits are incremented in set_ocs_aes_command()
552 * 11 - Full 128 bits are incremented in set_ocs_aes_command()
556 iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); in set_ocs_aes_command()
567 /* Set endianness recommended by data-sheet. */ in ocs_aes_init()
575 * Write the byte length of the last AES/SM4 block of Payload data (without
593 iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); in ocs_aes_write_last_data_blk_len()
598 * If OK return 0; else return -EINVAL.
610 return -EINVAL; in ocs_aes_validate_inputs()
615 return -EINVAL; in ocs_aes_validate_inputs()
619 return -EINVAL; in ocs_aes_validate_inputs()
623 * to destination using DMA. in ocs_aes_validate_inputs()
625 * AES mode is irrelevant, but both source and destination DMA in ocs_aes_validate_inputs()
626 * linked-list must be defined. in ocs_aes_validate_inputs()
631 return -EINVAL; in ocs_aes_validate_inputs()
644 return -EINVAL; in ocs_aes_validate_inputs()
649 return -EINVAL; in ocs_aes_validate_inputs()
656 return -EINVAL; in ocs_aes_validate_inputs()
661 return -EINVAL; in ocs_aes_validate_inputs()
665 return -EINVAL; in ocs_aes_validate_inputs()
672 return -EINVAL; in ocs_aes_validate_inputs()
677 return -EINVAL; in ocs_aes_validate_inputs()
681 return -EINVAL; in ocs_aes_validate_inputs()
688 return -EINVAL; in ocs_aes_validate_inputs()
693 return -EINVAL; in ocs_aes_validate_inputs()
697 return -EINVAL; in ocs_aes_validate_inputs()
704 return -EINVAL; in ocs_aes_validate_inputs()
712 return -EINVAL; in ocs_aes_validate_inputs()
716 return -EINVAL; in ocs_aes_validate_inputs()
720 return -EINVAL; in ocs_aes_validate_inputs()
724 return -EINVAL; in ocs_aes_validate_inputs()
731 return -EINVAL; in ocs_aes_validate_inputs()
736 return -EINVAL; in ocs_aes_validate_inputs()
740 return -EINVAL; in ocs_aes_validate_inputs()
744 return -EINVAL; in ocs_aes_validate_inputs()
753 return -EINVAL; in ocs_aes_validate_inputs()
757 return -EINVAL; in ocs_aes_validate_inputs()
769 return -EINVAL; in ocs_aes_validate_inputs()
773 return -EINVAL; in ocs_aes_validate_inputs()
778 return -EINVAL; in ocs_aes_validate_inputs()
783 * ocs_aes_op() - Perform AES/SM4 operation.
788 * @dst_dma_list: The OCS DMA list mapping output memory.
789 * @src_dma_list: The OCS DMA list mapping input payload data.
819 return -EINVAL; in ocs_aes_op()
827 /* Write the byte length of the last data block to engine. */ in ocs_aes_op()
833 iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); in ocs_aes_op()
834 iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); in ocs_aes_op()
835 iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); in ocs_aes_op()
836 iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); in ocs_aes_op()
842 /* Configure and activate input / output DMA. */ in ocs_aes_op()
854 /* For all other modes, just write the 'termination' bit. */ in ocs_aes_op()
865 iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); in ocs_aes_op()
866 iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); in ocs_aes_op()
867 iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); in ocs_aes_op()
868 iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); in ocs_aes_op()
874 /* Compute and write J0 to engine registers. */
884 iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); in ocs_aes_gcm_write_j0()
885 iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); in ocs_aes_gcm_write_j0()
886 iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); in ocs_aes_gcm_write_j0()
887 iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); in ocs_aes_gcm_write_j0()
901 tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); in ocs_aes_gcm_read_tag()
902 tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); in ocs_aes_gcm_read_tag()
903 tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); in ocs_aes_gcm_read_tag()
904 tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); in ocs_aes_gcm_read_tag()
910 * ocs_aes_gcm_op() - Perform GCM operation.
914 * @dst_dma_list: The OCS DMA list mapping output memory.
915 * @src_dma_list: The OCS DMA list mapping input payload data.
918 * @aad_dma_list: The OCS DMA list mapping input AAD data.
951 /* Compute and write J0 to OCS HW. */ in ocs_aes_gcm_op()
954 /* Write out_tag byte length */ in ocs_aes_gcm_op()
955 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); in ocs_aes_gcm_op()
957 /* Write the byte length of the last plaintext / ciphertext block. */ in ocs_aes_gcm_op()
960 /* Write ciphertext bit length */ in ocs_aes_gcm_op()
963 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); in ocs_aes_gcm_op()
965 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); in ocs_aes_gcm_op()
967 /* Write aad bit length */ in ocs_aes_gcm_op()
970 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); in ocs_aes_gcm_op()
972 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); in ocs_aes_gcm_op()
979 /* If aad present, configure DMA to feed it to the engine. */ in ocs_aes_gcm_op()
986 /* Wait for DMA transfer to complete. */ in ocs_aes_gcm_op()
1000 /* Configure and activate DMA for both input and output data. */ in ocs_aes_gcm_op()
1022 /* Write encrypted tag to AES/SM4 engine. */
1028 /* Ensure DMA input buffer is empty */ in ocs_aes_ccm_write_encrypted_tag()
1033 * ciphertext before the tag is written. So delay needed after DMA has in ocs_aes_ccm_write_encrypted_tag()
1040 /* Write encrypted tag to AES/SM4 engine. */ in ocs_aes_ccm_write_encrypted_tag()
1042 iowrite8(in_tag[i], aes_dev->base_reg + in ocs_aes_ccm_write_encrypted_tag()
1048 * Write B0 CCM block to OCS AES HW.
1050 * Note: B0 format is documented in NIST Special Publication 800-38C
1051 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1068 * bit 5-3: t value encoded as (t-2)/2 in ocs_aes_ccm_write_b0()
1069 * bit 2-0: q value encoded as q - 1 in ocs_aes_ccm_write_b0()
1077 * encoded as (t - 2) / 2 in ocs_aes_ccm_write_b0()
1079 b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; in ocs_aes_ccm_write_b0()
1083 * q - 1 == iv[0] & 0x7; in ocs_aes_ccm_write_b0()
1087 * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] in ocs_aes_ccm_write_b0()
1088 * and must be copied to b0[1]..b0[15-q]. in ocs_aes_ccm_write_b0()
1092 for (i = 1; i <= 15 - q; i++) in ocs_aes_ccm_write_b0()
1096 * Q is encoded in q octets, in big-endian order, so to write it, we in ocs_aes_ccm_write_b0()
1099 i = sizeof(b0) - 1; in ocs_aes_ccm_write_b0()
1103 i--; in ocs_aes_ccm_write_b0()
1104 q--; in ocs_aes_ccm_write_b0()
1111 return -EOVERFLOW; in ocs_aes_ccm_write_b0()
1112 /* Now write B0 to OCS AES input buffer. */ in ocs_aes_ccm_write_b0()
1114 iowrite8(b0[i], aes_dev->base_reg + in ocs_aes_ccm_write_b0()
1120 * Write adata length to OCS AES HW.
1122 * Note: adata len encoding is documented in NIST Special Publication 800-38C
1123 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1134 * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets in ocs_aes_ccm_write_adata_len()
1136 * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, in ocs_aes_ccm_write_adata_len()
1155 aes_dev->base_reg + in ocs_aes_ccm_write_adata_len()
1173 * Form the encoding of the Associated data length and write it in ocs_aes_ccm_do_adata()
1178 /* Configure the AES/SM4 DMA to fetch the Associated Data */ in ocs_aes_ccm_do_adata()
1181 /* Activate DMA to fetch Associated data. */ in ocs_aes_ccm_do_adata()
1187 /* Wait for DMA transfer to complete. */ in ocs_aes_ccm_do_adata()
1207 * Configure and activate DMA for both input and output in ocs_aes_ccm_encrypt_do_payload()
1214 /* Configure and activate DMA for output data only. */ in ocs_aes_ccm_encrypt_do_payload()
1235 /* Let engine process 0-length input. */ in ocs_aes_ccm_decrypt_do_payload()
1244 * Configure and activate DMA for both input and output in ocs_aes_ccm_decrypt_do_payload()
1257 * Enable DMA DONE interrupt; once DMA transfer is over, in ocs_aes_ccm_decrypt_do_payload()
1278 tag[i] = ioread32(aes_dev->base_reg + in ccm_compare_tag_to_yr()
1280 yr[i] = ioread32(aes_dev->base_reg + in ccm_compare_tag_to_yr()
1285 return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; in ccm_compare_tag_to_yr()
1289 * ocs_aes_ccm_op() - Perform CCM operation.
1293 * @dst_dma_list: The OCS DMA list mapping output memory.
1294 * @src_dma_list: The OCS DMA list mapping input payload data.
1297 * @adata_dma_list: The OCS DMA list mapping input A-data.
1333 * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt in ocs_aes_ccm_op()
1345 aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); in ocs_aes_ccm_op()
1347 aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); in ocs_aes_ccm_op()
1349 aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); in ocs_aes_ccm_op()
1351 aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); in ocs_aes_ccm_op()
1353 /* Write MAC/tag length in register AES_TLEN */ in ocs_aes_ccm_op()
1354 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); in ocs_aes_ccm_op()
1356 * Write the byte length of the last AES/SM4 block of Payload data in ocs_aes_ccm_op()
1367 /* Form block B0 and write it to the AES/SM4 input buffer. */ in ocs_aes_ccm_op()
1402 * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
1404 * @sg: The SG list OCS DMA linked list will be created from. When
1407 * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
1409 * @dll_desc: The OCS DMA dma_list to use to store information about the
1412 * OCS DMA linked list.
1430 return -EINVAL; in ocs_create_linked_list_from_sg()
1433 dll_desc->vaddr = NULL; in ocs_create_linked_list_from_sg()
1434 dll_desc->dma_addr = DMA_MAPPING_ERROR; in ocs_create_linked_list_from_sg()
1435 dll_desc->size = 0; in ocs_create_linked_list_from_sg()
1442 data_offset -= sg_dma_len(sg); in ocs_create_linked_list_from_sg()
1443 sg_dma_count--; in ocs_create_linked_list_from_sg()
1447 return -EINVAL; in ocs_create_linked_list_from_sg()
1450 /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ in ocs_create_linked_list_from_sg()
1457 return -EINVAL; in ocs_create_linked_list_from_sg()
1463 return -EINVAL; in ocs_create_linked_list_from_sg()
1465 /* Allocate the DMA list, one entry for each SG entry. */ in ocs_create_linked_list_from_sg()
1466 dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; in ocs_create_linked_list_from_sg()
1467 dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, in ocs_create_linked_list_from_sg()
1468 &dll_desc->dma_addr, GFP_KERNEL); in ocs_create_linked_list_from_sg()
1469 if (!dll_desc->vaddr) in ocs_create_linked_list_from_sg()
1470 return -ENOMEM; in ocs_create_linked_list_from_sg()
1472 /* Populate DMA linked list entries. */ in ocs_create_linked_list_from_sg()
1473 ll = dll_desc->vaddr; in ocs_create_linked_list_from_sg()
1476 ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? in ocs_create_linked_list_from_sg()
1477 (sg_dma_len(sg) - data_offset) : data_size; in ocs_create_linked_list_from_sg()
1479 data_size -= ll[i].src_len; in ocs_create_linked_list_from_sg()
1480 /* Current element points to the DMA address of the next one. */ in ocs_create_linked_list_from_sg()
1481 ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); in ocs_create_linked_list_from_sg()
1485 ll[i - 1].next = 0; in ocs_create_linked_list_from_sg()
1486 ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; in ocs_create_linked_list_from_sg()