1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (C) ST-Ericsson AB 2013
21 #include <linux/dma-mapping.h>
33 /* Defaults used if virtio config space is unavailable */
41 /* struct cfv_napi_contxt - NAPI context info
46  *	  used to indicate invalid head-id.
53 /* struct cfv_stats - statistics for debugfs
58  * @tx_full_ring:	Number times TX ring was full
59  * @tx_no_mem:		Number of times TX went out of memory
60  * @tx_flow_on:		Number of flow on (TX)
61  * @tx_kicks:		Number of TX kicks
74 /* struct cfv_info - Caif Virtio control structure
78  * @vq_tx:	tx/uplink virtqueue
81  *		to reopen the tx-queues after overload.
83  * @tx_release_tasklet: Tasklet for freeing consumed TX buffers
92  * @allocsz:    size of dma memory reserved for TX buffers
93  * @alloc_addr: virtual address to dma memory for TX buffers
94  * @alloc_dma:  dma address to dma memory for TX buffers
95  * @genpool:    Gen Pool used for allocating TX buffers
129 /* struct buf_info - maintains transmit buffer data handle
142 	struct cfv_info *cfv = vq_tx->vdev->priv;  in cfv_release_cb()
144 	++cfv->stats.tx_kicks;  in cfv_release_cb()
145 	tasklet_schedule(&cfv->tx_release_tasklet);  in cfv_release_cb()
152 	gen_pool_free(cfv->genpool, (unsigned long) buf_info->vaddr,  in free_buf_info()
153 		      buf_info->size);  in free_buf_info()
158  * a TX msg we just sent, and the buffer is put back to the used ring.
162 	struct cfv_info *cfv = vq_tx->vdev->priv;  in cfv_release_used_buf()
165 	BUG_ON(vq_tx != cfv->vq_tx);  in cfv_release_used_buf()
172 		spin_lock_irqsave(&cfv->tx_lock, flags);  in cfv_release_used_buf()
174 		spin_unlock_irqrestore(&cfv->tx_lock, flags);  in cfv_release_used_buf()
182 		/* watermark_tx indicates if we previously stopped the tx  in cfv_release_used_buf()
183 		 * queues. If we have enough free stots in the virtio ring,  in cfv_release_used_buf()
184 		 * re-establish memory reserved and open up tx queues.  in cfv_release_used_buf()
186 		if (cfv->vq_tx->num_free <= cfv->watermark_tx)  in cfv_release_used_buf()
189 		/* Re-establish memory reserve */  in cfv_release_used_buf()
190 		if (cfv->reserved_mem == 0 && cfv->genpool)  in cfv_release_used_buf()
191 			cfv->reserved_mem =  in cfv_release_used_buf()
192 				gen_pool_alloc(cfv->genpool,  in cfv_release_used_buf()
193 					       cfv->reserved_size);  in cfv_release_used_buf()
195 		/* Open up the tx queues */  in cfv_release_used_buf()
196 		if (cfv->reserved_mem) {  in cfv_release_used_buf()
197 			cfv->watermark_tx =  in cfv_release_used_buf()
198 				virtqueue_get_vring_size(cfv->vq_tx);  in cfv_release_used_buf()
199 			netif_tx_wake_all_queues(cfv->ndev);  in cfv_release_used_buf()
201 			 * disable notifications when queues are opened.  in cfv_release_used_buf()
203 			virtqueue_disable_cb(cfv->vq_tx);  in cfv_release_used_buf()
204 			++cfv->stats.tx_flow_on;  in cfv_release_used_buf()
207 			WARN_ON(cfv->watermark_tx >  in cfv_release_used_buf()
208 			       virtqueue_get_vring_size(cfv->vq_tx));  in cfv_release_used_buf()
209 			cfv->watermark_tx +=  in cfv_release_used_buf()
210 				virtqueue_get_vring_size(cfv->vq_tx) / 4;  in cfv_release_used_buf()
224 	/* Verify that packet size with down-link header and mtu size */  in cfv_alloc_and_copy_skb()
225 	if (frm_len > cfv->mru || frm_len <= cfv->rx_hr + cfv->rx_tr) {  in cfv_alloc_and_copy_skb()
226 		netdev_err(cfv->ndev,  in cfv_alloc_and_copy_skb()
228 			   frm_len, cfv->mru,  cfv->rx_hr,  in cfv_alloc_and_copy_skb()
229 			   cfv->rx_tr);  in cfv_alloc_and_copy_skb()
230 		*err = -EPROTO;  in cfv_alloc_and_copy_skb()
234 	cfpkt_len = frm_len - (cfv->rx_hr + cfv->rx_tr);  in cfv_alloc_and_copy_skb()
235 	pad_len = (unsigned long)(frm + cfv->rx_hr) & (IP_HDR_ALIGN - 1);  in cfv_alloc_and_copy_skb()
237 	skb = netdev_alloc_skb(cfv->ndev, frm_len + pad_len);  in cfv_alloc_and_copy_skb()
239 		*err = -ENOMEM;  in cfv_alloc_and_copy_skb()
243 	skb_reserve(skb, cfv->rx_hr + pad_len);  in cfv_alloc_and_copy_skb()
245 	skb_put_data(skb, frm + cfv->rx_hr, cfpkt_len);  in cfv_alloc_and_copy_skb()
257 	struct vringh_kiov *riov = &cfv->ctx.riov;  in cfv_rx_poll()
266 		if (riov->i == riov->used) {  in cfv_rx_poll()
267 			if (cfv->ctx.head != USHRT_MAX) {  in cfv_rx_poll()
268 				vringh_complete_kern(cfv->vr_rx,  in cfv_rx_poll()
269 						     cfv->ctx.head,  in cfv_rx_poll()
271 				cfv->ctx.head = USHRT_MAX;  in cfv_rx_poll()
275 				cfv->vr_rx,  in cfv_rx_poll()
278 				&cfv->ctx.head,  in cfv_rx_poll()
285 		buf = phys_to_virt((unsigned long) riov->iov[riov->i].iov_base);  in cfv_rx_poll()
289 					     riov->iov[riov->i].iov_len);  in cfv_rx_poll()
294 		skb_len = skb->len;  in cfv_rx_poll()
295 		skb->protocol = htons(ETH_P_CAIF);  in cfv_rx_poll()
297 		skb->dev = cfv->ndev;  in cfv_rx_poll()
300 			++cfv->ndev->stats.rx_dropped;  in cfv_rx_poll()
302 			++cfv->ndev->stats.rx_packets;  in cfv_rx_poll()
303 			cfv->ndev->stats.rx_bytes += skb_len;  in cfv_rx_poll()
306 		++riov->i;  in cfv_rx_poll()
310 	++cfv->stats.rx_napi_resched;  in cfv_rx_poll()
316 		++cfv->stats.rx_napi_complete;  in cfv_rx_poll()
320 		if (unlikely(!vringh_notify_enable_kern(cfv->vr_rx)) &&  in cfv_rx_poll()
322 			vringh_notify_disable_kern(cfv->vr_rx);  in cfv_rx_poll()
327 	case -ENOMEM:  in cfv_rx_poll()
328 		++cfv->stats.rx_nomem;  in cfv_rx_poll()
332 		vringh_notify_enable_kern(cfv->vr_rx);  in cfv_rx_poll()
337 		netdev_warn(cfv->ndev, "Bad ring, disable device\n");  in cfv_rx_poll()
338 		cfv->ndev->stats.rx_dropped = riov->used - riov->i;  in cfv_rx_poll()
340 		vringh_notify_disable_kern(cfv->vr_rx);  in cfv_rx_poll()
341 		netif_carrier_off(cfv->ndev);  in cfv_rx_poll()
345 	if (rxcnt && vringh_need_notify_kern(cfv->vr_rx) > 0)  in cfv_rx_poll()
346 		vringh_notify(cfv->vr_rx);  in cfv_rx_poll()
352 	struct cfv_info *cfv = vdev->priv;  in cfv_recv()
354 	++cfv->stats.rx_kicks;  in cfv_recv()
355 	vringh_notify_disable_kern(cfv->vr_rx);  in cfv_recv()
356 	napi_schedule(&cfv->napi);  in cfv_recv()
361 	if (cfv->alloc_addr)  in cfv_destroy_genpool()
362 		dma_free_coherent(cfv->vdev->dev.parent->parent,  in cfv_destroy_genpool()
363 				  cfv->allocsz, cfv->alloc_addr,  in cfv_destroy_genpool()
364 				  cfv->alloc_dma);  in cfv_destroy_genpool()
366 	if (!cfv->genpool)  in cfv_destroy_genpool()
368 	gen_pool_free(cfv->genpool,  cfv->reserved_mem,  in cfv_destroy_genpool()
369 		      cfv->reserved_size);  in cfv_destroy_genpool()
370 	gen_pool_destroy(cfv->genpool);  in cfv_destroy_genpool()
371 	cfv->genpool = NULL;  in cfv_destroy_genpool()
383 	err = -ENOMEM;  in cfv_create_genpool()
384 	cfv->allocsz = (virtqueue_get_vring_size(cfv->vq_tx) *  in cfv_create_genpool()
385 			(ETH_DATA_LEN + cfv->tx_hr + cfv->tx_tr) * 11)/10;  in cfv_create_genpool()
386 	if (cfv->allocsz <= (num_possible_cpus() + 1) * cfv->ndev->mtu)  in cfv_create_genpool()
387 		return -EINVAL;  in cfv_create_genpool()
390 		if (cfv->allocsz <= num_possible_cpus() * cfv->ndev->mtu) {  in cfv_create_genpool()
391 			netdev_info(cfv->ndev, "Not enough device memory\n");  in cfv_create_genpool()
392 			return -ENOMEM;  in cfv_create_genpool()
395 		cfv->alloc_addr = dma_alloc_coherent(  in cfv_create_genpool()
396 						cfv->vdev->dev.parent->parent,  in cfv_create_genpool()
397 						cfv->allocsz, &cfv->alloc_dma,  in cfv_create_genpool()
399 		if (cfv->alloc_addr)  in cfv_create_genpool()
402 		cfv->allocsz = (cfv->allocsz * 3) >> 2;  in cfv_create_genpool()
405 	netdev_dbg(cfv->ndev, "Allocated %zd bytes from dma-memory\n",  in cfv_create_genpool()
406 		   cfv->allocsz);  in cfv_create_genpool()
409 	cfv->genpool = gen_pool_create(7, -1);  in cfv_create_genpool()
410 	if (!cfv->genpool)  in cfv_create_genpool()
413 	err = gen_pool_add_virt(cfv->genpool, (unsigned long)cfv->alloc_addr,  in cfv_create_genpool()
414 				(phys_addr_t)virt_to_phys(cfv->alloc_addr),  in cfv_create_genpool()
415 				cfv->allocsz, -1);  in cfv_create_genpool()
420 	 * in the memory pool, we stop TX flow and release the reserve.  in cfv_create_genpool()
422 	cfv->reserved_size = num_possible_cpus() * cfv->ndev->mtu;  in cfv_create_genpool()
423 	cfv->reserved_mem = gen_pool_alloc(cfv->genpool,  in cfv_create_genpool()
424 					   cfv->reserved_size);  in cfv_create_genpool()
425 	if (!cfv->reserved_mem) {  in cfv_create_genpool()
426 		err = -ENOMEM;  in cfv_create_genpool()
430 	cfv->watermark_tx = virtqueue_get_vring_size(cfv->vq_tx);  in cfv_create_genpool()
437 /* Enable the CAIF interface and allocate the memory-pool */
443 		return -ENOMEM;  in cfv_netdev_open()
446 	napi_enable(&cfv->napi);  in cfv_netdev_open()
449 	napi_schedule(&cfv->napi);  in cfv_netdev_open()
453 /* Disable the CAIF interface and free the memory-pool */
460 	/* Disable interrupts, queues and NAPI polling */  in cfv_netdev_close()
462 	virtqueue_disable_cb(cfv->vq_tx);  in cfv_netdev_close()
463 	vringh_notify_disable_kern(cfv->vr_rx);  in cfv_netdev_close()
464 	napi_disable(&cfv->napi);  in cfv_netdev_close()
466 	/* Release any TX buffers on both used and available rings */  in cfv_netdev_close()
467 	cfv_release_used_buf(cfv->vq_tx);  in cfv_netdev_close()
468 	spin_lock_irqsave(&cfv->tx_lock, flags);  in cfv_netdev_close()
469 	while ((buf_info = virtqueue_detach_unused_buf(cfv->vq_tx)))  in cfv_netdev_close()
471 	spin_unlock_irqrestore(&cfv->tx_lock, flags);  in cfv_netdev_close()
478 /* Allocate a buffer in dma-memory and copy skb to it */
483 	struct caif_payload_info *info = (void *)&skb->cb;  in cfv_alloc_and_copy_to_shm()
487 	if (!cfv->genpool)  in cfv_alloc_and_copy_to_shm()
490 	if (unlikely(cfv->tx_hr + skb->len + cfv->tx_tr > cfv->mtu)) {  in cfv_alloc_and_copy_to_shm()
491 		netdev_warn(cfv->ndev, "Invalid packet len (%d > %d)\n",  in cfv_alloc_and_copy_to_shm()
492 			    cfv->tx_hr + skb->len + cfv->tx_tr, cfv->mtu);  in cfv_alloc_and_copy_to_shm()
501 	hdr_ofs = cfv->tx_hr + info->hdr_len;  in cfv_alloc_and_copy_to_shm()
502 	pad_len = hdr_ofs & (IP_HDR_ALIGN - 1);  in cfv_alloc_and_copy_to_shm()
503 	buf_info->size = cfv->tx_hr + skb->len + cfv->tx_tr + pad_len;  in cfv_alloc_and_copy_to_shm()
506 	buf_info->vaddr = (void *)gen_pool_alloc(cfv->genpool, buf_info->size);  in cfv_alloc_and_copy_to_shm()
507 	if (unlikely(!buf_info->vaddr))  in cfv_alloc_and_copy_to_shm()
511 	skb_copy_bits(skb, 0, buf_info->vaddr + cfv->tx_hr + pad_len, skb->len);  in cfv_alloc_and_copy_to_shm()
512 	sg_init_one(sg, buf_info->vaddr + pad_len,  in cfv_alloc_and_copy_to_shm()
513 		    skb->len + cfv->tx_hr + cfv->rx_hr);  in cfv_alloc_and_copy_to_shm()
532 	cfv_release_used_buf(cfv->vq_tx);  in cfv_netdev_tx()
533 	spin_lock_irqsave(&cfv->tx_lock, flags);  in cfv_netdev_tx()
535 	/* Flow-off check takes into account number of cpus to make sure  in cfv_netdev_tx()
538 	 * Flow-on is triggered when sufficient buffers are freed  in cfv_netdev_tx()
540 	if (unlikely(cfv->vq_tx->num_free <= num_present_cpus())) {  in cfv_netdev_tx()
542 		cfv->stats.tx_full_ring++;  in cfv_netdev_tx()
550 		cfv->stats.tx_no_mem++;  in cfv_netdev_tx()
553 		if (cfv->reserved_mem && cfv->genpool) {  in cfv_netdev_tx()
554 			gen_pool_free(cfv->genpool,  cfv->reserved_mem,  in cfv_netdev_tx()
555 				      cfv->reserved_size);  in cfv_netdev_tx()
556 			cfv->reserved_mem = 0;  in cfv_netdev_tx()
563 		cfv->watermark_tx = virtqueue_get_vring_size(cfv->vq_tx) / 4;  in cfv_netdev_tx()
564 		/* Enable notifications of recycled TX buffers */  in cfv_netdev_tx()
565 		virtqueue_enable_cb(cfv->vq_tx);  in cfv_netdev_tx()
571 		netdev_warn(cfv->ndev, "Out of gen_pool memory\n");  in cfv_netdev_tx()
575 	ret = virtqueue_add_outbuf(cfv->vq_tx, &sg, 1, buf_info, GFP_ATOMIC);  in cfv_netdev_tx()
578 		netdev_warn(cfv->ndev, "Failed adding buffer to TX vring:%d\n",  in cfv_netdev_tx()
584 	cfv->ndev->stats.tx_packets++;  in cfv_netdev_tx()
585 	cfv->ndev->stats.tx_bytes += skb->len;  in cfv_netdev_tx()
586 	spin_unlock_irqrestore(&cfv->tx_lock, flags);  in cfv_netdev_tx()
589 	virtqueue_kick(cfv->vq_tx);  in cfv_netdev_tx()
594 	spin_unlock_irqrestore(&cfv->tx_lock, flags);  in cfv_netdev_tx()
595 	cfv->ndev->stats.tx_dropped++;  in cfv_netdev_tx()
604 	cfv_release_used_buf(cfv->vq_tx);  in cfv_tx_release_tasklet()
615 	netdev->netdev_ops = &cfv_netdev_ops;  in cfv_netdev_setup()
616 	netdev->type = ARPHRD_CAIF;  in cfv_netdev_setup()
617 	netdev->tx_queue_len = 100;  in cfv_netdev_setup()
618 	netdev->flags = IFF_POINTOPOINT | IFF_NOARP;  in cfv_netdev_setup()
619 	netdev->mtu = CFV_DEF_MTU_SIZE;  in cfv_netdev_setup()
620 	netdev->needs_free_netdev = true;  in cfv_netdev_setup()
626 	cfv->debugfs = debugfs_create_dir(netdev_name(cfv->ndev), NULL);  in debugfs_init()
628 	debugfs_create_u32("rx-napi-complete", 0400, cfv->debugfs,  in debugfs_init()
629 			   &cfv->stats.rx_napi_complete);  in debugfs_init()
630 	debugfs_create_u32("rx-napi-resched", 0400, cfv->debugfs,  in debugfs_init()
631 			   &cfv->stats.rx_napi_resched);  in debugfs_init()
632 	debugfs_create_u32("rx-nomem", 0400, cfv->debugfs,  in debugfs_init()
633 			   &cfv->stats.rx_nomem);  in debugfs_init()
634 	debugfs_create_u32("rx-kicks", 0400, cfv->debugfs,  in debugfs_init()
635 			   &cfv->stats.rx_kicks);  in debugfs_init()
636 	debugfs_create_u32("tx-full-ring", 0400, cfv->debugfs,  in debugfs_init()
637 			   &cfv->stats.tx_full_ring);  in debugfs_init()
638 	debugfs_create_u32("tx-no-mem", 0400, cfv->debugfs,  in debugfs_init()
639 			   &cfv->stats.tx_no_mem);  in debugfs_init()
640 	debugfs_create_u32("tx-kicks", 0400, cfv->debugfs,  in debugfs_init()
641 			   &cfv->stats.tx_kicks);  in debugfs_init()
642 	debugfs_create_u32("tx-flow-on", 0400, cfv->debugfs,  in debugfs_init()
643 			   &cfv->stats.tx_flow_on);  in debugfs_init()
658 		return -ENOMEM;  in cfv_probe()
661 	cfv->vdev = vdev;  in cfv_probe()
662 	cfv->ndev = netdev;  in cfv_probe()
664 	spin_lock_init(&cfv->tx_lock);  in cfv_probe()
667 	err = -ENODEV;  in cfv_probe()
668 	if (!vdev->vringh_config || !vdev->vringh_config->find_vrhs)  in cfv_probe()
671 	err = vdev->vringh_config->find_vrhs(vdev, 1, &cfv->vr_rx, &vrh_cbs);  in cfv_probe()
675 	/* Get the TX virtio ring. This is a "guest side vring". */  in cfv_probe()
676 	cfv->vq_tx = virtio_find_single_vq(vdev, cfv_release_cb, "output");  in cfv_probe()
677 	if (IS_ERR(cfv->vq_tx)) {  in cfv_probe()
678 		err = PTR_ERR(cfv->vq_tx);  in cfv_probe()
682 	/* Get the CAIF configuration from virtio config space, if available */  in cfv_probe()
683 	if (vdev->config->get) {  in cfv_probe()
685 			     &cfv->tx_hr);  in cfv_probe()
687 			     &cfv->rx_hr);  in cfv_probe()
689 			     &cfv->tx_tr);  in cfv_probe()
691 			     &cfv->rx_tr);  in cfv_probe()
693 			     &cfv->mtu);  in cfv_probe()
695 			     &cfv->mru);  in cfv_probe()
697 		cfv->tx_hr = CFV_DEF_HEADROOM;  in cfv_probe()
698 		cfv->rx_hr = CFV_DEF_HEADROOM;  in cfv_probe()
699 		cfv->tx_tr = CFV_DEF_TAILROOM;  in cfv_probe()
700 		cfv->rx_tr = CFV_DEF_TAILROOM;  in cfv_probe()
701 		cfv->mtu = CFV_DEF_MTU_SIZE;  in cfv_probe()
702 		cfv->mru = CFV_DEF_MTU_SIZE;  in cfv_probe()
705 	netdev->needed_headroom = cfv->tx_hr;  in cfv_probe()
706 	netdev->needed_tailroom = cfv->tx_tr;  in cfv_probe()
708 	/* Disable buffer release interrupts unless we have stopped TX queues */  in cfv_probe()
709 	virtqueue_disable_cb(cfv->vq_tx);  in cfv_probe()
711 	netdev->mtu = cfv->mtu - cfv->tx_tr;  in cfv_probe()
712 	vdev->priv = cfv;  in cfv_probe()
715 	vringh_kiov_init(&cfv->ctx.riov, NULL, 0);  in cfv_probe()
716 	cfv->ctx.head = USHRT_MAX;  in cfv_probe()
717 	netif_napi_add_weight(netdev, &cfv->napi, cfv_rx_poll,  in cfv_probe()
720 	tasklet_setup(&cfv->tx_release_tasklet, cfv_tx_release_tasklet);  in cfv_probe()
732 		dev_err(&vdev->dev, "Unable to register netdev (%d)\n", err);  in cfv_probe()
744 	netdev_warn(cfv->ndev, "CAIF Virtio probe failed:%d\n", err);  in cfv_probe()
746 	if (cfv->vr_rx)  in cfv_probe()
747 		vdev->vringh_config->del_vrhs(cfv->vdev);  in cfv_probe()
748 	if (cfv->vdev)  in cfv_probe()
749 		vdev->config->del_vqs(cfv->vdev);  in cfv_probe()
756 	struct cfv_info *cfv = vdev->priv;  in cfv_remove()
759 	dev_close(cfv->ndev);  in cfv_remove()
762 	tasklet_kill(&cfv->tx_release_tasklet);  in cfv_remove()
763 	debugfs_remove_recursive(cfv->debugfs);  in cfv_remove()
765 	vringh_kiov_cleanup(&cfv->ctx.riov);  in cfv_remove()
767 	vdev->vringh_config->del_vrhs(cfv->vdev);  in cfv_remove()
768 	cfv->vr_rx = NULL;  in cfv_remove()
769 	vdev->config->del_vqs(cfv->vdev);  in cfv_remove()
770 	unregister_netdev(cfv->ndev);  in cfv_remove()