13  * Copyright (c) 2010-2014 Freescale Semiconductor, Inc.
45  *	RAID Engine (RE) block is capable of offloading XOR, memcpy and P/Q
67 #include <linux/dma-mapping.h>
86 /* Add descriptors into per chan software queue - submit_q */
95 	re_chan = container_of(tx->chan, struct fsl_re_chan, chan);  in fsl_re_tx_submit()
97 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_tx_submit()
99 	list_add_tail(&desc->node, &re_chan->submit_q);  in fsl_re_tx_submit()
100 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_tx_submit()
115 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_issue_pending()
117 		in_be32(&re_chan->jrregs->inbring_slot_avail));  in fsl_re_issue_pending()
119 	list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) {  in fsl_re_issue_pending()
123 		list_move_tail(&desc->node, &re_chan->active_q);  in fsl_re_issue_pending()
125 		memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count],  in fsl_re_issue_pending()
126 		       &desc->hwdesc, sizeof(struct fsl_re_hw_desc));  in fsl_re_issue_pending()
128 		re_chan->inb_count = (re_chan->inb_count + 1) &  in fsl_re_issue_pending()
130 		out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1));  in fsl_re_issue_pending()
131 		avail--;  in fsl_re_issue_pending()
133 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_issue_pending()
138 	dma_cookie_complete(&desc->async_tx);  in fsl_re_desc_done()
139 	dma_descriptor_unmap(&desc->async_tx);  in fsl_re_desc_done()
140 	dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);  in fsl_re_desc_done()
148 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_cleanup_descs()
149 	list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) {  in fsl_re_cleanup_descs()
150 		if (async_tx_test_ack(&desc->async_tx))  in fsl_re_cleanup_descs()
151 			list_move_tail(&desc->node, &re_chan->free_q);  in fsl_re_cleanup_descs()
153 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_cleanup_descs()
155 	fsl_re_issue_pending(&re_chan->chan);  in fsl_re_cleanup_descs()
169 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_dequeue()
170 	count =	FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full));  in fsl_re_dequeue()
171 	while (count--) {  in fsl_re_dequeue()
173 		hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count];  in fsl_re_dequeue()
174 		list_for_each_entry_safe(desc, _desc, &re_chan->active_q,  in fsl_re_dequeue()
177 			if (desc->hwdesc.lbea32 == hwdesc->lbea32 &&  in fsl_re_dequeue()
178 			    desc->hwdesc.addr_low == hwdesc->addr_low) {  in fsl_re_dequeue()
186 			list_move_tail(&desc->node, &re_chan->ack_q);  in fsl_re_dequeue()
188 			dev_err(re_chan->dev,  in fsl_re_dequeue()
192 		oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK;  in fsl_re_dequeue()
193 		re_chan->oub_count = oub_count;  in fsl_re_dequeue()
195 		out_be32(&re_chan->jrregs->oubring_job_rmvd,  in fsl_re_dequeue()
198 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_dequeue()
209 	irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status);  in fsl_re_isr()
219 		status = in_be32(&re_chan->jrregs->jr_status);  in fsl_re_isr()
220 		dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n",  in fsl_re_isr()
225 	out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR);  in fsl_re_isr()
227 	tasklet_schedule(&re_chan->irqtask);  in fsl_re_isr()
254 	desc->re_chan = re_chan;  in fsl_re_init_desc()
255 	desc->async_tx.tx_submit = fsl_re_tx_submit;  in fsl_re_init_desc()
256 	dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);  in fsl_re_init_desc()
257 	INIT_LIST_HEAD(&desc->node);  in fsl_re_init_desc()
259 	desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT;  in fsl_re_init_desc()
260 	desc->hwdesc.lbea32 = upper_32_bits(paddr);  in fsl_re_init_desc()
261 	desc->hwdesc.addr_low = lower_32_bits(paddr);  in fsl_re_init_desc()
262 	desc->cf_addr = cf;  in fsl_re_init_desc()
263 	desc->cf_paddr = paddr;  in fsl_re_init_desc()
265 	desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE);  in fsl_re_init_desc()
266 	desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE;  in fsl_re_init_desc()
281 	spin_lock_irqsave(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
282 	if (!list_empty(&re_chan->free_q)) {  in fsl_re_chan_alloc_desc()
284 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_chan_alloc_desc()
286 		list_del(&desc->node);  in fsl_re_chan_alloc_desc()
288 		desc->async_tx.flags = flags;  in fsl_re_chan_alloc_desc()
290 	spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
297 		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT,  in fsl_re_chan_alloc_desc()
305 		desc->async_tx.flags = flags;  in fsl_re_chan_alloc_desc()
307 		spin_lock_irqsave(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
308 		re_chan->alloc_count++;  in fsl_re_chan_alloc_desc()
309 		spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
322 	struct fsl_re_xor_cdb *xor;  in fsl_re_prep_dma_genq()  local
331 		dev_err(re_chan->dev, "genq tx length %zu, max length %d\n",  in fsl_re_prep_dma_genq()
345 	/* Filling xor CDB */  in fsl_re_prep_dma_genq()
347 	cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT;  in fsl_re_prep_dma_genq()
351 	xor = desc->cdb_addr;  in fsl_re_prep_dma_genq()
352 	xor->cdb32 = cdb;  in fsl_re_prep_dma_genq()
357 			xor->gfm[i] = scf[i];  in fsl_re_prep_dma_genq()
359 			xor->gfm[i++] = 1;  in fsl_re_prep_dma_genq()
361 		/* compute P, that is XOR all srcs */  in fsl_re_prep_dma_genq()
363 			xor->gfm[i] = 1;  in fsl_re_prep_dma_genq()
367 	cf = desc->cf_addr;  in fsl_re_prep_dma_genq()
368 	fill_cfd_frame(cf, 0, sizeof(*xor), desc->cdb_paddr, 0);  in fsl_re_prep_dma_genq()
381 	cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;  in fsl_re_prep_dma_genq()
383 	return &desc->async_tx;  in fsl_re_prep_dma_genq()
388  * XOR calculation is called GenQ calculation done through GenQ command
418 		dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n",  in fsl_re_prep_dma_pq()
466 	cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT;  in fsl_re_prep_dma_pq()
471 	pq = desc->cdb_addr;  in fsl_re_prep_dma_pq()
472 	pq->cdb32 = cdb;  in fsl_re_prep_dma_pq()
474 	p = pq->gfm_q1;  in fsl_re_prep_dma_pq()
488 	cf = desc->cf_addr;  in fsl_re_prep_dma_pq()
489 	fill_cfd_frame(cf, 0, sizeof(struct fsl_re_pq_cdb), desc->cdb_paddr, 0);  in fsl_re_prep_dma_pq()
501 		if (src_cnt - save_src_cnt == 3) {  in fsl_re_prep_dma_pq()
509 			dev_err(re_chan->dev, "PQ tx continuation error!\n");  in fsl_re_prep_dma_pq()
515 	cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;  in fsl_re_prep_dma_pq()
517 	return &desc->async_tx;  in fsl_re_prep_dma_pq()
539 		dev_err(re_chan->dev, "cp tx length is %zu, max length is %d\n",  in fsl_re_prep_dma_memcpy()
554 	move = desc->cdb_addr;  in fsl_re_prep_dma_memcpy()
555 	move->cdb32 = cdb;  in fsl_re_prep_dma_memcpy()
558 	cf = desc->cf_addr;  in fsl_re_prep_dma_memcpy()
559 	fill_cfd_frame(cf, 0, sizeof(*move), desc->cdb_paddr, 0);  in fsl_re_prep_dma_memcpy()
569 	return &desc->async_tx;  in fsl_re_prep_dma_memcpy()
586 		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL,  in fsl_re_alloc_chan_resources()
593 		INIT_LIST_HEAD(&desc->node);  in fsl_re_alloc_chan_resources()
596 		list_add_tail(&desc->node, &re_chan->free_q);  in fsl_re_alloc_chan_resources()
597 		re_chan->alloc_count++;  in fsl_re_alloc_chan_resources()
599 	return re_chan->alloc_count;  in fsl_re_alloc_chan_resources()
608 	while (re_chan->alloc_count--) {  in fsl_re_free_chan_resources()
609 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_free_chan_resources()
613 		list_del(&desc->node);  in fsl_re_free_chan_resources()
614 		dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr,  in fsl_re_free_chan_resources()
615 			      desc->cf_paddr);  in fsl_re_free_chan_resources()
619 	if (!list_empty(&re_chan->free_q))  in fsl_re_free_chan_resources()
620 		dev_err(re_chan->dev, "chan resource cannot be cleaned!\n");  in fsl_re_free_chan_resources()
635 	dev = &ofdev->dev;  in fsl_re_chan_probe()
637 	dma_dev = &re_priv->dma_dev;  in fsl_re_chan_probe()
641 		return -ENOMEM;  in fsl_re_chan_probe()
647 		ret = -EINVAL;  in fsl_re_chan_probe()
655 		ret = -ENODEV;  in fsl_re_chan_probe()
659 	chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs +  in fsl_re_chan_probe()
663 	chan->irq = irq_of_parse_and_map(np, 0);  in fsl_re_chan_probe()
664 	if (!chan->irq) {  in fsl_re_chan_probe()
666 		ret = -ENODEV;  in fsl_re_chan_probe()
670 	snprintf(chan->name, sizeof(chan->name), "re_jr%02d", q);  in fsl_re_chan_probe()
672 	chandev = &chan_ofdev->dev;  in fsl_re_chan_probe()
673 	tasklet_setup(&chan->irqtask, fsl_re_dequeue);  in fsl_re_chan_probe()
675 	ret = request_irq(chan->irq, fsl_re_isr, 0, chan->name, chandev);  in fsl_re_chan_probe()
678 		ret = -EINVAL;  in fsl_re_chan_probe()
682 	re_priv->re_jrs[q] = chan;  in fsl_re_chan_probe()
683 	chan->chan.device = dma_dev;  in fsl_re_chan_probe()
684 	chan->chan.private = chan;  in fsl_re_chan_probe()
685 	chan->dev = chandev;  in fsl_re_chan_probe()
686 	chan->re_dev = re_priv;  in fsl_re_chan_probe()
688 	spin_lock_init(&chan->desc_lock);  in fsl_re_chan_probe()
689 	INIT_LIST_HEAD(&chan->ack_q);  in fsl_re_chan_probe()
690 	INIT_LIST_HEAD(&chan->active_q);  in fsl_re_chan_probe()
691 	INIT_LIST_HEAD(&chan->submit_q);  in fsl_re_chan_probe()
692 	INIT_LIST_HEAD(&chan->free_q);  in fsl_re_chan_probe()
694 	chan->inb_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool,  in fsl_re_chan_probe()
695 		GFP_KERNEL, &chan->inb_phys_addr);  in fsl_re_chan_probe()
696 	if (!chan->inb_ring_virt_addr) {  in fsl_re_chan_probe()
698 		ret = -ENOMEM;  in fsl_re_chan_probe()
702 	chan->oub_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool,  in fsl_re_chan_probe()
703 		GFP_KERNEL, &chan->oub_phys_addr);  in fsl_re_chan_probe()
704 	if (!chan->oub_ring_virt_addr) {  in fsl_re_chan_probe()
706 		ret = -ENOMEM;  in fsl_re_chan_probe()
711 	out_be32(&chan->jrregs->inbring_base_h,  in fsl_re_chan_probe()
712 		 chan->inb_phys_addr & FSL_RE_ADDR_BIT_MASK);  in fsl_re_chan_probe()
713 	out_be32(&chan->jrregs->oubring_base_h,  in fsl_re_chan_probe()
714 		 chan->oub_phys_addr & FSL_RE_ADDR_BIT_MASK);  in fsl_re_chan_probe()
715 	out_be32(&chan->jrregs->inbring_base_l,  in fsl_re_chan_probe()
716 		 chan->inb_phys_addr >> FSL_RE_ADDR_BIT_SHIFT);  in fsl_re_chan_probe()
717 	out_be32(&chan->jrregs->oubring_base_l,  in fsl_re_chan_probe()
718 		 chan->oub_phys_addr >> FSL_RE_ADDR_BIT_SHIFT);  in fsl_re_chan_probe()
719 	out_be32(&chan->jrregs->inbring_size,  in fsl_re_chan_probe()
721 	out_be32(&chan->jrregs->oubring_size,  in fsl_re_chan_probe()
724 	/* Read LIODN value from u-boot */  in fsl_re_chan_probe()
725 	status = in_be32(&chan->jrregs->jr_config_1) & FSL_RE_REG_LIODN_MASK;  in fsl_re_chan_probe()
728 	out_be32(&chan->jrregs->jr_config_1,  in fsl_re_chan_probe()
734 	out_be32(&chan->jrregs->jr_command, FSL_RE_ENABLE);  in fsl_re_chan_probe()
739 	dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr,  in fsl_re_chan_probe()
740 		      chan->inb_phys_addr);  in fsl_re_chan_probe()
756 	struct device *dev = &ofdev->dev;  in fsl_re_probe()
760 		return -ENOMEM;  in fsl_re_probe()
764 		return -ENODEV;  in fsl_re_probe()
767 	re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res));  in fsl_re_probe()
768 	if (!re_priv->re_regs)  in fsl_re_probe()
769 		return -EBUSY;  in fsl_re_probe()
772 	out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE);  in fsl_re_probe()
775 	out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY);  in fsl_re_probe()
778 		 in_be32(&re_priv->re_regs->re_version_id),  in fsl_re_probe()
779 		 in_be32(&re_priv->re_regs->global_config),  in fsl_re_probe()
780 		 in_be32(&re_priv->re_regs->galois_field_config));  in fsl_re_probe()
782 	dma_dev = &re_priv->dma_dev;  in fsl_re_probe()
783 	dma_dev->dev = dev;  in fsl_re_probe()
784 	INIT_LIST_HEAD(&dma_dev->channels);  in fsl_re_probe()
787 	dma_dev->device_alloc_chan_resources = fsl_re_alloc_chan_resources;  in fsl_re_probe()
788 	dma_dev->device_tx_status = fsl_re_tx_status;  in fsl_re_probe()
789 	dma_dev->device_issue_pending = fsl_re_issue_pending;  in fsl_re_probe()
791 	dma_dev->max_xor = FSL_RE_MAX_XOR_SRCS;  in fsl_re_probe()
792 	dma_dev->device_prep_dma_xor = fsl_re_prep_dma_xor;  in fsl_re_probe()
793 	dma_cap_set(DMA_XOR, dma_dev->cap_mask);  in fsl_re_probe()
795 	dma_dev->max_pq = FSL_RE_MAX_PQ_SRCS;  in fsl_re_probe()
796 	dma_dev->device_prep_dma_pq = fsl_re_prep_dma_pq;  in fsl_re_probe()
797 	dma_cap_set(DMA_PQ, dma_dev->cap_mask);  in fsl_re_probe()
799 	dma_dev->device_prep_dma_memcpy = fsl_re_prep_dma_memcpy;  in fsl_re_probe()
800 	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);  in fsl_re_probe()
802 	dma_dev->device_free_chan_resources = fsl_re_free_chan_resources;  in fsl_re_probe()
804 	re_priv->total_chans = 0;  in fsl_re_probe()
806 	re_priv->cf_desc_pool = dmam_pool_create("fsl_re_cf_desc_pool", dev,  in fsl_re_probe()
810 	if (!re_priv->cf_desc_pool) {  in fsl_re_probe()
812 		return -ENOMEM;  in fsl_re_probe()
815 	re_priv->hw_desc_pool = dmam_pool_create("fsl_re_hw_desc_pool", dev,  in fsl_re_probe()
818 	if (!re_priv->hw_desc_pool) {  in fsl_re_probe()
820 		return -ENOMEM;  in fsl_re_probe()
826 	for_each_compatible_node(np, NULL, "fsl,raideng-v1.0-job-queue") {  in fsl_re_probe()
831 			return -ENODEV;  in fsl_re_probe()
836 					     "fsl,raideng-v1.0-job-ring");  in fsl_re_probe()
839 				re_priv->total_chans++;  in fsl_re_probe()
851 	tasklet_kill(&chan->irqtask);  in fsl_re_remove_chan()
853 	dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr,  in fsl_re_remove_chan()
854 		      chan->inb_phys_addr);  in fsl_re_remove_chan()
856 	dma_pool_free(chan->re_dev->hw_desc_pool, chan->oub_ring_virt_addr,  in fsl_re_remove_chan()
857 		      chan->oub_phys_addr);  in fsl_re_remove_chan()
866 	dev = &ofdev->dev;  in fsl_re_remove()
870 	for (i = 0; i < re_priv->total_chans; i++)  in fsl_re_remove()
871 		fsl_re_remove_chan(re_priv->re_jrs[i]);  in fsl_re_remove()
874 	dma_async_device_unregister(&re_priv->dma_dev);  in fsl_re_remove()
878 	{ .compatible = "fsl,raideng-v1.0", },
885 		.name = "fsl-raideng",
895 MODULE_LICENSE("GPL v2");