1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <linux/dma-mapping.h>
17 #include "ccp-dev.h"
19 /* Allocate the requested number of contiguous LSB slots
20  * from the LSB bitmap. Look in the private range for this
21  * queue first; failing that, check the public area.
23  * Return: first slot number
30 	/* First look at the map for the queue */  in ccp_lsb_alloc()
31 	if (cmd_q->lsb >= 0) {  in ccp_lsb_alloc()
32 		start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,  in ccp_lsb_alloc()
36 			bitmap_set(cmd_q->lsbmap, start, count);  in ccp_lsb_alloc()
37 			return start + cmd_q->lsb * LSB_SIZE;  in ccp_lsb_alloc()
42 	ccp = cmd_q->ccp;  in ccp_lsb_alloc()
44 		mutex_lock(&ccp->sb_mutex);  in ccp_lsb_alloc()
46 		start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,  in ccp_lsb_alloc()
51 			bitmap_set(ccp->lsbmap, start, count);  in ccp_lsb_alloc()
53 			mutex_unlock(&ccp->sb_mutex);  in ccp_lsb_alloc()
57 		ccp->sb_avail = 0;  in ccp_lsb_alloc()
59 		mutex_unlock(&ccp->sb_mutex);  in ccp_lsb_alloc()
62 		if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))  in ccp_lsb_alloc()
67 /* Free a number of LSB slots from the bitmap, starting at
76 	if (cmd_q->lsb == start) {  in ccp_lsb_free()
77 		/* An entry from the private LSB */  in ccp_lsb_free()
78 		bitmap_clear(cmd_q->lsbmap, start, count);  in ccp_lsb_free()
81 		struct ccp_device *ccp = cmd_q->ccp;  in ccp_lsb_free()
83 		mutex_lock(&ccp->sb_mutex);  in ccp_lsb_free()
84 		bitmap_clear(ccp->lsbmap, start, count);  in ccp_lsb_free()
85 		ccp->sb_avail = 1;  in ccp_lsb_free()
86 		mutex_unlock(&ccp->sb_mutex);  in ccp_lsb_free()
87 		wake_up_interruptible_all(&ccp->sb_queue);  in ccp_lsb_free()
137 #define	CCP_AES_SIZE(p)		((p)->aes.size)
138 #define	CCP_AES_ENCRYPT(p)	((p)->aes.encrypt)
139 #define	CCP_AES_MODE(p)		((p)->aes.mode)
140 #define	CCP_AES_TYPE(p)		((p)->aes.type)
141 #define	CCP_XTS_SIZE(p)		((p)->aes_xts.size)
142 #define	CCP_XTS_TYPE(p)		((p)->aes_xts.type)
143 #define	CCP_XTS_ENCRYPT(p)	((p)->aes_xts.encrypt)
144 #define	CCP_DES3_SIZE(p)	((p)->des3.size)
145 #define	CCP_DES3_ENCRYPT(p)	((p)->des3.encrypt)
146 #define	CCP_DES3_MODE(p)	((p)->des3.mode)
147 #define	CCP_DES3_TYPE(p)	((p)->des3.type)
148 #define	CCP_SHA_TYPE(p)		((p)->sha.type)
149 #define	CCP_RSA_SIZE(p)		((p)->rsa.size)
150 #define	CCP_PT_BYTESWAP(p)	((p)->pt.byteswap)
151 #define	CCP_PT_BITWISE(p)	((p)->pt.bitwise)
152 #define	CCP_ECC_MODE(p)		((p)->ecc.mode)
153 #define	CCP_ECC_AFFINE(p)	((p)->ecc.one)
156 #define CCP5_CMD_DW0(p)		((p)->dw0)
166 #define CCP5_CMD_DW1(p)		((p)->length)
170 #define CCP5_CMD_DW2(p)		((p)->src_lo)
174 #define CCP5_CMD_DW3(p)		((p)->dw3)
175 #define CCP5_CMD_SRC_MEM(p)	((p)->dw3.src_mem)
176 #define CCP5_CMD_SRC_HI(p)	((p)->dw3.src_hi)
177 #define CCP5_CMD_LSB_ID(p)	((p)->dw3.lsb_cxt_id)
178 #define CCP5_CMD_FIX_SRC(p)	((p)->dw3.fixed)
181 #define CCP5_CMD_DW4(p)		((p)->dw4)
183 #define CCP5_CMD_DW5(p)		((p)->dw5.fields.dst_hi)
185 #define CCP5_CMD_DST_MEM(p)	((p)->dw5.fields.dst_mem)
186 #define CCP5_CMD_FIX_DST(p)	((p)->dw5.fields.fixed)
187 #define CCP5_CMD_SHA_LO(p)	((p)->dw4.sha_len_lo)
188 #define CCP5_CMD_SHA_HI(p)	((p)->dw5.sha_len_hi)
191 #define CCP5_CMD_DW6(p)		((p)->key_lo)
193 #define CCP5_CMD_DW7(p)		((p)->dw7)
194 #define CCP5_CMD_KEY_HI(p)	((p)->dw7.key_hi)
195 #define CCP5_CMD_KEY_MEM(p)	((p)->dw7.key_mem)
212 	queue_start = low_address(cmd_q->qdma_tail);  in ccp5_get_free_slots()
213 	head_lo = ioread32(cmd_q->reg_head_lo);  in ccp5_get_free_slots()
214 	head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);  in ccp5_get_free_slots()
216 	n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;  in ccp5_get_free_slots()
230 	cmd_q->total_ops++;  in ccp5_do_cmd()
236 	mutex_lock(&cmd_q->q_mutex);  in ccp5_do_cmd()
238 	mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx];  in ccp5_do_cmd()
243 	cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;  in ccp5_do_cmd()
249 	tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);  in ccp5_do_cmd()
250 	iowrite32(tail, cmd_q->reg_tail_lo);  in ccp5_do_cmd()
253 	iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);  in ccp5_do_cmd()
254 	mutex_unlock(&cmd_q->q_mutex);  in ccp5_do_cmd()
258 		ret = wait_event_interruptible(cmd_q->int_queue,  in ccp5_do_cmd()
259 					       cmd_q->int_rcvd);  in ccp5_do_cmd()
260 		if (ret || cmd_q->cmd_error) {  in ccp5_do_cmd()
264 			if (cmd_q->cmd_error)  in ccp5_do_cmd()
265 				ccp_log_error(cmd_q->ccp,  in ccp5_do_cmd()
266 					      cmd_q->cmd_error);  in ccp5_do_cmd()
267 			iowrite32(tail, cmd_q->reg_head_lo);  in ccp5_do_cmd()
269 				ret = -EIO;  in ccp5_do_cmd()
271 		cmd_q->int_rcvd = 0;  in ccp5_do_cmd()
281 	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;  in ccp5_perform_aes()
283 	op->cmd_q->total_aes_ops++;  in ccp5_perform_aes()
290 	CCP5_CMD_SOC(&desc) = op->soc;  in ccp5_perform_aes()
292 	CCP5_CMD_INIT(&desc) = op->init;  in ccp5_perform_aes()
293 	CCP5_CMD_EOM(&desc) = op->eom;  in ccp5_perform_aes()
297 	CCP_AES_ENCRYPT(&function) = op->u.aes.action;  in ccp5_perform_aes()
298 	CCP_AES_MODE(&function) = op->u.aes.mode;  in ccp5_perform_aes()
299 	CCP_AES_TYPE(&function) = op->u.aes.type;  in ccp5_perform_aes()
300 	CCP_AES_SIZE(&function) = op->u.aes.size;  in ccp5_perform_aes()
304 	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;  in ccp5_perform_aes()
306 	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);  in ccp5_perform_aes()
307 	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);  in ccp5_perform_aes()
310 	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);  in ccp5_perform_aes()
311 	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);  in ccp5_perform_aes()
317 	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;  in ccp5_perform_aes()
319 	return ccp5_do_cmd(&desc, op->cmd_q);  in ccp5_perform_aes()
326 	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;  in ccp5_perform_xts_aes()
328 	op->cmd_q->total_xts_aes_ops++;  in ccp5_perform_xts_aes()
335 	CCP5_CMD_SOC(&desc) = op->soc;  in ccp5_perform_xts_aes()
337 	CCP5_CMD_INIT(&desc) = op->init;  in ccp5_perform_xts_aes()
338 	CCP5_CMD_EOM(&desc) = op->eom;  in ccp5_perform_xts_aes()
342 	CCP_XTS_TYPE(&function) = op->u.xts.type;  in ccp5_perform_xts_aes()
343 	CCP_XTS_ENCRYPT(&function) = op->u.xts.action;  in ccp5_perform_xts_aes()
344 	CCP_XTS_SIZE(&function) = op->u.xts.unit_size;  in ccp5_perform_xts_aes()
347 	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;  in ccp5_perform_xts_aes()
349 	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);  in ccp5_perform_xts_aes()
350 	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);  in ccp5_perform_xts_aes()
353 	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);  in ccp5_perform_xts_aes()
354 	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);  in ccp5_perform_xts_aes()
360 	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;  in ccp5_perform_xts_aes()
362 	return ccp5_do_cmd(&desc, op->cmd_q);  in ccp5_perform_xts_aes()
370 	op->cmd_q->total_sha_ops++;  in ccp5_perform_sha()
377 	CCP5_CMD_SOC(&desc) = op->soc;  in ccp5_perform_sha()
380 	CCP5_CMD_EOM(&desc) = op->eom;  in ccp5_perform_sha()
384 	CCP_SHA_TYPE(&function) = op->u.sha.type;  in ccp5_perform_sha()
387 	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;  in ccp5_perform_sha()
389 	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);  in ccp5_perform_sha()
390 	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);  in ccp5_perform_sha()
393 	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;  in ccp5_perform_sha()
395 	if (op->eom) {  in ccp5_perform_sha()
396 		CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);  in ccp5_perform_sha()
397 		CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);  in ccp5_perform_sha()
403 	return ccp5_do_cmd(&desc, op->cmd_q);  in ccp5_perform_sha()
410 	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;  in ccp5_perform_des3()
412 	op->cmd_q->total_3des_ops++;  in ccp5_perform_des3()
419 	CCP5_CMD_SOC(&desc) = op->soc;  in ccp5_perform_des3()
421 	CCP5_CMD_INIT(&desc) = op->init;  in ccp5_perform_des3()
422 	CCP5_CMD_EOM(&desc) = op->eom;  in ccp5_perform_des3()
426 	CCP_DES3_ENCRYPT(&function) = op->u.des3.action;  in ccp5_perform_des3()
427 	CCP_DES3_MODE(&function) = op->u.des3.mode;  in ccp5_perform_des3()
428 	CCP_DES3_TYPE(&function) = op->u.des3.type;  in ccp5_perform_des3()
431 	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;  in ccp5_perform_des3()
433 	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);  in ccp5_perform_des3()
434 	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);  in ccp5_perform_des3()
437 	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);  in ccp5_perform_des3()
438 	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);  in ccp5_perform_des3()
444 	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;  in ccp5_perform_des3()
446 	return ccp5_do_cmd(&desc, op->cmd_q);  in ccp5_perform_des3()
454 	op->cmd_q->total_rsa_ops++;  in ccp5_perform_rsa()
461 	CCP5_CMD_SOC(&desc) = op->soc;  in ccp5_perform_rsa()
468 	CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;  in ccp5_perform_rsa()
471 	CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;  in ccp5_perform_rsa()
474 	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);  in ccp5_perform_rsa()
475 	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);  in ccp5_perform_rsa()
479 	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);  in ccp5_perform_rsa()
480 	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);  in ccp5_perform_rsa()
484 	CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);  in ccp5_perform_rsa()
485 	CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);  in ccp5_perform_rsa()
488 	return ccp5_do_cmd(&desc, op->cmd_q);  in ccp5_perform_rsa()
495 	struct ccp_dma_info *saddr = &op->src.u.dma;  in ccp5_perform_passthru()
496 	struct ccp_dma_info *daddr = &op->dst.u.dma;  in ccp5_perform_passthru()
499 	op->cmd_q->total_pt_ops++;  in ccp5_perform_passthru()
508 	CCP5_CMD_EOM(&desc) = op->eom;  in ccp5_perform_passthru()
512 	CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;  in ccp5_perform_passthru()
513 	CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;  in ccp5_perform_passthru()
517 	if (op->src.type == CCP_MEMTYPE_SYSTEM)  in ccp5_perform_passthru()
518 		CCP5_CMD_LEN(&desc) = saddr->length;  in ccp5_perform_passthru()
520 		CCP5_CMD_LEN(&desc) = daddr->length;  in ccp5_perform_passthru()
522 	if (op->src.type == CCP_MEMTYPE_SYSTEM) {  in ccp5_perform_passthru()
523 		CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);  in ccp5_perform_passthru()
524 		CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);  in ccp5_perform_passthru()
527 		if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)  in ccp5_perform_passthru()
528 			CCP5_CMD_LSB_ID(&desc) = op->sb_key;  in ccp5_perform_passthru()
530 		u32 key_addr = op->src.u.sb * CCP_SB_BYTES;  in ccp5_perform_passthru()
537 	if (op->dst.type == CCP_MEMTYPE_SYSTEM) {  in ccp5_perform_passthru()
538 		CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);  in ccp5_perform_passthru()
539 		CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);  in ccp5_perform_passthru()
542 		u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;  in ccp5_perform_passthru()
549 	return ccp5_do_cmd(&desc, op->cmd_q);  in ccp5_perform_passthru()
557 	op->cmd_q->total_ecc_ops++;  in ccp5_perform_ecc()
571 	function.ecc.mode = op->u.ecc.function;  in ccp5_perform_ecc()
574 	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;  in ccp5_perform_ecc()
576 	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);  in ccp5_perform_ecc()
577 	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);  in ccp5_perform_ecc()
580 	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);  in ccp5_perform_ecc()
581 	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);  in ccp5_perform_ecc()
584 	return ccp5_do_cmd(&desc, op->cmd_q);  in ccp5_perform_ecc()
589 	int q_mask = 1 << cmd_q->id;  in ccp_find_lsb_regions()
598 			bitmap_set(cmd_q->lsbmask, j, 1);  in ccp_find_lsb_regions()
601 	queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);  in ccp_find_lsb_regions()
602 	dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",  in ccp_find_lsb_regions()
603 		 cmd_q->id, queues);  in ccp_find_lsb_regions()
605 	return queues ? 0 : -EINVAL;  in ccp_find_lsb_regions()
627 	for (i = 0; i < ccp->cmd_q_count; i++) {  in ccp_find_and_assign_lsb_to_q()
628 		struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];  in ccp_find_and_assign_lsb_to_q()
630 		qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);  in ccp_find_and_assign_lsb_to_q()
633 			bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);  in ccp_find_and_assign_lsb_to_q()
638 					/* We found an available LSB  in ccp_find_and_assign_lsb_to_q()
641 					cmd_q->lsb = bitno;  in ccp_find_and_assign_lsb_to_q()
643 					dev_dbg(ccp->dev,  in ccp_find_and_assign_lsb_to_q()
644 						 "Queue %d gets LSB %d\n",  in ccp_find_and_assign_lsb_to_q()
652 				return -EINVAL;  in ccp_find_and_assign_lsb_to_q()
653 			n_lsbs--;  in ccp_find_and_assign_lsb_to_q()
659 /* For each queue, from the most- to least-constrained:
660  * find an LSB that can be assigned to the queue. If there are N queues that
662  * dedicated LSB. Remaining LSB regions become a shared resource.
663  * If we have fewer LSBs than queues, all LSB regions become shared resources.
677 	for (i = 0; i < ccp->cmd_q_count; i++)  in ccp_assign_lsbs()
679 			  lsb_pub, ccp->cmd_q[i].lsbmask,  in ccp_assign_lsbs()
684 	if (n_lsbs >= ccp->cmd_q_count) {  in ccp_assign_lsbs()
685 		/* We have enough LSBS to give every queue a private LSB.  in ccp_assign_lsbs()
687 		 * constrained in LSB choice. When an LSB is privately  in ccp_assign_lsbs()
697 				return -EINVAL;  in ccp_assign_lsbs()
704 	 * shared. Any zero bits in the lsb_pub mask represent an LSB region  in ccp_assign_lsbs()
705 	 * that can't be used as a shared resource, so mark the LSB slots for  in ccp_assign_lsbs()
712 		bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);  in ccp_assign_lsbs()
724 	for (i = 0; i < ccp->cmd_q_count; i++)  in ccp5_disable_queue_interrupts()
725 		iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);  in ccp5_disable_queue_interrupts()
732 	for (i = 0; i < ccp->cmd_q_count; i++)  in ccp5_enable_queue_interrupts()
733 		iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);  in ccp5_enable_queue_interrupts()
742 	for (i = 0; i < ccp->cmd_q_count; i++) {  in ccp5_irq_bh()
743 		struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];  in ccp5_irq_bh()
745 		status = ioread32(cmd_q->reg_interrupt_status);  in ccp5_irq_bh()
748 			cmd_q->int_status = status;  in ccp5_irq_bh()
749 			cmd_q->q_status = ioread32(cmd_q->reg_status);  in ccp5_irq_bh()
750 			cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);  in ccp5_irq_bh()
752 			/* On error, only save the first error value */  in ccp5_irq_bh()
753 			if ((status & INT_ERROR) && !cmd_q->cmd_error)  in ccp5_irq_bh()
754 				cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);  in ccp5_irq_bh()
756 			cmd_q->int_rcvd = 1;  in ccp5_irq_bh()
759 			iowrite32(status, cmd_q->reg_interrupt_status);  in ccp5_irq_bh()
760 			wake_up_interruptible(&cmd_q->int_queue);  in ccp5_irq_bh()
771 	ccp->total_interrupts++;  in ccp5_irq_handler()
772 	if (ccp->use_tasklet)  in ccp5_irq_handler()
773 		tasklet_schedule(&ccp->irq_tasklet);  in ccp5_irq_handler()
781 	struct device *dev = ccp->dev;  in ccp5_init()
791 	qmr = ioread32(ccp->io_regs + Q_MASK_REG);  in ccp5_init()
804 	for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {  in ccp5_init()
810 			 ccp->name, i);  in ccp5_init()
816 			ret = -ENOMEM;  in ccp5_init()
820 		cmd_q = &ccp->cmd_q[ccp->cmd_q_count];  in ccp5_init()
821 		ccp->cmd_q_count++;  in ccp5_init()
823 		cmd_q->ccp = ccp;  in ccp5_init()
824 		cmd_q->id = i;  in ccp5_init()
825 		cmd_q->dma_pool = dma_pool;  in ccp5_init()
826 		mutex_init(&cmd_q->q_mutex);  in ccp5_init()
830 		cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);  in ccp5_init()
831 		cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize,  in ccp5_init()
832 						   &cmd_q->qbase_dma,  in ccp5_init()
834 		if (!cmd_q->qbase) {  in ccp5_init()
836 			ret = -ENOMEM;  in ccp5_init()
840 		cmd_q->qidx = 0;  in ccp5_init()
844 		cmd_q->reg_control = ccp->io_regs +  in ccp5_init()
846 		cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;  in ccp5_init()
847 		cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;  in ccp5_init()
848 		cmd_q->reg_int_enable = cmd_q->reg_control +  in ccp5_init()
850 		cmd_q->reg_interrupt_status = cmd_q->reg_control +  in ccp5_init()
852 		cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;  in ccp5_init()
853 		cmd_q->reg_int_status = cmd_q->reg_control +  in ccp5_init()
855 		cmd_q->reg_dma_status = cmd_q->reg_control +  in ccp5_init()
857 		cmd_q->reg_dma_read_status = cmd_q->reg_control +  in ccp5_init()
859 		cmd_q->reg_dma_write_status = cmd_q->reg_control +  in ccp5_init()
862 		init_waitqueue_head(&cmd_q->int_queue);  in ccp5_init()
867 	if (ccp->cmd_q_count == 0) {  in ccp5_init()
875 	for (i = 0; i < ccp->cmd_q_count; i++) {  in ccp5_init()
876 		cmd_q = &ccp->cmd_q[i];  in ccp5_init()
878 		cmd_q->qcontrol = 0; /* Start with nothing */  in ccp5_init()
879 		iowrite32(cmd_q->qcontrol, cmd_q->reg_control);  in ccp5_init()
881 		ioread32(cmd_q->reg_int_status);  in ccp5_init()
882 		ioread32(cmd_q->reg_status);  in ccp5_init()
885 		iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);  in ccp5_init()
890 	ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);  in ccp5_init()
896 	if (ccp->use_tasklet)  in ccp5_init()
897 		tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,  in ccp5_init()
900 	dev_dbg(dev, "Loading LSB map...\n");  in ccp5_init()
901 	/* Copy the private LSB mask to the public registers */  in ccp5_init()
902 	status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);  in ccp5_init()
903 	status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);  in ccp5_init()
904 	iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);  in ccp5_init()
905 	iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);  in ccp5_init()
910 	for (i = 0; i < ccp->cmd_q_count; i++) {  in ccp5_init()
914 		cmd_q = &ccp->cmd_q[i];  in ccp5_init()
916 		cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);  in ccp5_init()
917 		cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;  in ccp5_init()
919 		cmd_q->qdma_tail = cmd_q->qbase_dma;  in ccp5_init()
920 		dma_addr_lo = low_address(cmd_q->qdma_tail);  in ccp5_init()
921 		iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);  in ccp5_init()
922 		iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);  in ccp5_init()
924 		dma_addr_hi = high_address(cmd_q->qdma_tail);  in ccp5_init()
925 		cmd_q->qcontrol |= (dma_addr_hi << 16);  in ccp5_init()
926 		iowrite32(cmd_q->qcontrol, cmd_q->reg_control);  in ccp5_init()
928 		/* Find the LSB regions accessible to the queue */  in ccp5_init()
930 		cmd_q->lsb = -1; /* Unassigned value */  in ccp5_init()
940 	/* Optimization: pre-allocate LSB slots for each queue */  in ccp5_init()
941 	for (i = 0; i < ccp->cmd_q_count; i++) {  in ccp5_init()
942 		ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);  in ccp5_init()
943 		ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);  in ccp5_init()
948 	for (i = 0; i < ccp->cmd_q_count; i++) {  in ccp5_init()
951 		cmd_q = &ccp->cmd_q[i];  in ccp5_init()
954 				      "%s-q%u", ccp->name, cmd_q->id);  in ccp5_init()
962 		cmd_q->kthread = kthread;  in ccp5_init()
992 	for (i = 0; i < ccp->cmd_q_count; i++)  in ccp5_init()
993 		if (ccp->cmd_q[i].kthread)  in ccp5_init()
994 			kthread_stop(ccp->cmd_q[i].kthread);  in ccp5_init()
997 	sp_free_ccp_irq(ccp->sp, ccp);  in ccp5_init()
1000 	for (i = 0; i < ccp->cmd_q_count; i++)  in ccp5_init()
1001 		dma_pool_destroy(ccp->cmd_q[i].dma_pool);  in ccp5_init()
1018 	/* Remove this device from the list of available units first */  in ccp5_destroy()
1031 	for (i = 0; i < ccp->cmd_q_count; i++) {  in ccp5_destroy()
1032 		cmd_q = &ccp->cmd_q[i];  in ccp5_destroy()
1035 		iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);  in ccp5_destroy()
1038 		iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);  in ccp5_destroy()
1039 		ioread32(cmd_q->reg_int_status);  in ccp5_destroy()
1040 		ioread32(cmd_q->reg_status);  in ccp5_destroy()
1044 	for (i = 0; i < ccp->cmd_q_count; i++)  in ccp5_destroy()
1045 		if (ccp->cmd_q[i].kthread)  in ccp5_destroy()
1046 			kthread_stop(ccp->cmd_q[i].kthread);  in ccp5_destroy()
1048 	sp_free_ccp_irq(ccp->sp, ccp);  in ccp5_destroy()
1051 	while (!list_empty(&ccp->cmd)) {  in ccp5_destroy()
1053 		cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);  in ccp5_destroy()
1054 		list_del(&cmd->entry);  in ccp5_destroy()
1055 		cmd->callback(cmd->data, -ENODEV);  in ccp5_destroy()
1057 	while (!list_empty(&ccp->backlog)) {  in ccp5_destroy()
1059 		cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);  in ccp5_destroy()
1060 		list_del(&cmd->entry);  in ccp5_destroy()
1061 		cmd->callback(cmd->data, -ENODEV);  in ccp5_destroy()
1068 	iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);  in ccp5_config()
1078 	iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);  in ccp5other_config()
1079 	iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);  in ccp5other_config()
1081 		rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);  in ccp5other_config()
1082 		iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);  in ccp5other_config()
1085 	iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);  in ccp5other_config()
1086 	iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);  in ccp5other_config()
1087 	iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);  in ccp5other_config()
1089 	iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);  in ccp5other_config()
1090 	iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);  in ccp5other_config()
1092 	iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);  in ccp5other_config()