1 // SPDX-License-Identifier: GPL-2.0
3  * Virtio-based remote processor messaging bus
8  * Ohad Ben-Cohen <ohad@wizery.com>
14 #include <linux/dma-mapping.h>
20 #include <linux/rpmsg.h>
21 #include <linux/rpmsg/byteorder.h>
22 #include <linux/rpmsg/ns.h>
34  * struct virtproc_info - virtual remote processor state
52  * This structure stores the rpmsg state of a given virtio remote processor
71 /* The feature bitmap for virtio rpmsg */
75  * struct rpmsg_hdr - common header for all rpmsg messages
79  * @len: length of payload (in bytes)
83  * Every message sent(/received) on the rpmsg bus begins with this header.
96  * struct virtio_rpmsg_channel - rpmsg channel descriptor
97  * @rpdev: the rpmsg channel device
100  * This structure stores the channel that links the rpmsg device to the virtio
115  * by the vring, upto a maximum of 512 buffers (256 in each direction).
122  * We might also want to add support for user-provided buffers in time.
124  * to achieve zero-copy messaging.
126  * Note that these numbers are purely a decision of this driver - we
127  * can change this without changing anything in the firmware of the remote
134  * Local addresses are dynamically allocated on-demand.
136  * in order to reserve that address range for predefined services.
167  * rpmsg_sg_init - initialize scatterlist according to cpu address location
173  * location (in vmalloc or in kernel).
189  * __ept_release() - deallocate an rpmsg endpoint
216 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
222 	kref_init(&ept->refcount);  in __rpmsg_create_ept()
223 	mutex_init(&ept->cb_lock);  in __rpmsg_create_ept()
225 	ept->rpdev = rpdev;  in __rpmsg_create_ept()
226 	ept->cb = cb;  in __rpmsg_create_ept()
227 	ept->priv = priv;  in __rpmsg_create_ept()
228 	ept->ops = &virtio_endpoint_ops;  in __rpmsg_create_ept()
239 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
241 	/* bind the endpoint to an rpmsg address (and allocate one if needed) */  in __rpmsg_create_ept()
242 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
247 	ept->addr = id;  in __rpmsg_create_ept()
249 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
254 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
255 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_create_ept()
263 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_create_channel()
272 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_release_channel()
274 	return rpmsg_unregister_device(&vrp->vdev->dev, chinfo);  in virtio_rpmsg_release_channel()
284 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
288  * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint
293  * bound to an rpmsg channel. This is needed for handling the internal
294  * name service endpoint, which isn't bound to an rpmsg channel.
301 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
302 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
303 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
305 	/* make sure in-flight inbound messages won't invoke cb anymore */  in __rpmsg_destroy_ept()
306 	mutex_lock(&ept->cb_lock);  in __rpmsg_destroy_ept()
307 	ept->cb = NULL;  in __rpmsg_destroy_ept()
308 	mutex_unlock(&ept->cb_lock);  in __rpmsg_destroy_ept()
310 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_destroy_ept()
315 	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev);  in virtio_rpmsg_destroy_ept()
317 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
323 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()
324 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_create()
328 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_create()
329 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
332 		strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name));  in virtio_rpmsg_announce_create()
333 		nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr);  in virtio_rpmsg_announce_create()
336 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_create()
347 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()
348 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_destroy()
352 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_destroy()
353 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
356 		strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name));  in virtio_rpmsg_announce_destroy()
357 		nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr);  in virtio_rpmsg_announce_destroy()
360 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_destroy()
381 	kfree(rpdev->driver_override);  in virtio_rpmsg_release_device()
386  * create an rpmsg channel using its name and address info.
395 	struct device *tmp, *dev = &vrp->vdev->dev;  in __rpmsg_create_channel()
404 				chinfo->name, chinfo->src, chinfo->dst);  in __rpmsg_create_channel()
413 	vch->vrp = vrp;  in __rpmsg_create_channel()
416 	rpdev = &vch->rpdev;  in __rpmsg_create_channel()
417 	rpdev->src = chinfo->src;  in __rpmsg_create_channel()
418 	rpdev->dst = chinfo->dst;  in __rpmsg_create_channel()
419 	rpdev->ops = &virtio_rpmsg_ops;  in __rpmsg_create_channel()
420 	rpdev->little_endian = virtio_is_little_endian(vrp->vdev);  in __rpmsg_create_channel()
423 	 * rpmsg server channels has predefined local address (for now),  in __rpmsg_create_channel()
426 	rpdev->announce = rpdev->src != RPMSG_ADDR_ANY;  in __rpmsg_create_channel()
428 	strscpy(rpdev->id.name, chinfo->name, sizeof(rpdev->id.name));  in __rpmsg_create_channel()
430 	rpdev->dev.parent = &vrp->vdev->dev;  in __rpmsg_create_channel()
431 	rpdev->dev.release = virtio_rpmsg_release_device;  in __rpmsg_create_channel()
446 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
452 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
453 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
456 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
458 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
464  * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed
475  * buffer is consumed (we rely on virtio's tx callback in order
482 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
485 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
486 		/* enable "tx-complete" interrupts before dozing off */  in rpmsg_upref_sleepers()
487 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
489 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
493  * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed
509 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
512 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
513 		/* disable "tx-complete" interrupts */  in rpmsg_downref_sleepers()
514 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
516 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
520  * rpmsg_send_offchannel_raw() - send a message across to the remote processor
521  * @rpdev: the rpmsg channel
526  * @wait: indicates whether caller should block in case no TX buffers available
528  * This function is the base implementation for all of the rpmsg sending API.
539  * sleep indefinitely due to misbehaving remote processors), and in that
540  * case -ERESTARTSYS is returned. The number '15' itself was picked
541  * arbitrarily; there's little point in asking drivers to provide a timeout
545  * the function will immediately fail, and -ENOMEM will be returned.
549  * (see include/linux/rpmsg.h).
558 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()
559 	struct device *dev = &rpdev->dev;  in rpmsg_send_offchannel_raw()
567 		return -EINVAL;  in rpmsg_send_offchannel_raw()
571 	 * We currently use fixed-sized buffers, and therefore the payload  in rpmsg_send_offchannel_raw()
575 	 * user-provided buffers (and then we can also support zero-copy  in rpmsg_send_offchannel_raw()
577 	 * variable-length buffer sizes.  in rpmsg_send_offchannel_raw()
579 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
581 		return -EMSGSIZE;  in rpmsg_send_offchannel_raw()
587 		return -ENOMEM;  in rpmsg_send_offchannel_raw()
591 		/* enable "tx-complete" interrupts, if not already enabled */  in rpmsg_send_offchannel_raw()
597 		 * little point in asking drivers to specify that.  in rpmsg_send_offchannel_raw()
600 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
604 		/* disable "tx-complete" interrupts if we're the last sleeper */  in rpmsg_send_offchannel_raw()
610 			return -ERESTARTSYS;  in rpmsg_send_offchannel_raw()
614 	msg->len = cpu_to_rpmsg16(rpdev, len);  in rpmsg_send_offchannel_raw()
615 	msg->flags = 0;  in rpmsg_send_offchannel_raw()
616 	msg->src = cpu_to_rpmsg32(rpdev, src);  in rpmsg_send_offchannel_raw()
617 	msg->dst = cpu_to_rpmsg32(rpdev, dst);  in rpmsg_send_offchannel_raw()
618 	msg->reserved = 0;  in rpmsg_send_offchannel_raw()
619 	memcpy(msg->data, data, len);  in rpmsg_send_offchannel_raw()
622 		src, dst, len, msg->flags, msg->reserved);  in rpmsg_send_offchannel_raw()
630 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
633 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
637 		 * (memory won't leak, but rpmsg won't use it again for TX).  in rpmsg_send_offchannel_raw()
645 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
647 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
653 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_send()
654 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_send()
662 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_sendto()
663 	u32 src = ept->addr;  in virtio_rpmsg_sendto()
671 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_send_offchannel()
678 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysend()
679 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_trysend()
687 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysendto()
688 	u32 src = ept->addr;  in virtio_rpmsg_trysendto()
696 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysend_offchannel()
703 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_get_mtu()
706 	return vch->vrp->buf_size - sizeof(struct rpmsg_hdr);  in virtio_rpmsg_get_mtu()
714 	bool little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_recv_single()
715 	unsigned int msg_len = __rpmsg16_to_cpu(little_endian, msg->len);  in rpmsg_recv_single()
719 		__rpmsg32_to_cpu(little_endian, msg->src),  in rpmsg_recv_single()
720 		__rpmsg32_to_cpu(little_endian, msg->dst), msg_len,  in rpmsg_recv_single()
721 		__rpmsg16_to_cpu(little_endian, msg->flags),  in rpmsg_recv_single()
722 		__rpmsg32_to_cpu(little_endian, msg->reserved));  in rpmsg_recv_single()
729 	 * We currently use fixed-sized buffers, so trivially sanitize  in rpmsg_recv_single()
732 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
733 	    msg_len > (len - sizeof(struct rpmsg_hdr))) {  in rpmsg_recv_single()
735 		return -EINVAL;  in rpmsg_recv_single()
739 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
741 	ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst));  in rpmsg_recv_single()
745 		kref_get(&ept->refcount);  in rpmsg_recv_single()
747 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
750 		/* make sure ept->cb doesn't go away while we use it */  in rpmsg_recv_single()
751 		mutex_lock(&ept->cb_lock);  in rpmsg_recv_single()
753 		if (ept->cb)  in rpmsg_recv_single()
754 			ept->cb(ept->rpdev, msg->data, msg_len, ept->priv,  in rpmsg_recv_single()
755 				__rpmsg32_to_cpu(little_endian, msg->src));  in rpmsg_recv_single()
757 		mutex_unlock(&ept->cb_lock);  in rpmsg_recv_single()
760 		kref_put(&ept->refcount, __ept_release);  in rpmsg_recv_single()
765 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
768 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
780 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()
781 	struct device *dev = &rvq->vdev->dev;  in rpmsg_recv_done()
806 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
813  * Normally, though, we suppress this "tx complete" interrupt in order to
818 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()
820 	dev_dbg(&svq->vdev->dev, "%s\n", __func__);  in rpmsg_xmit_done()
823 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
828  * create endpoint-to-endpoint communication without associated RPMsg channel.
829  * The endpoints are rattached to the ctrldev RPMsg device.
833 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_virtio_add_ctrl_dev()
840 		return ERR_PTR(-ENOMEM);  in rpmsg_virtio_add_ctrl_dev()
843 	vch->vrp = vrp;  in rpmsg_virtio_add_ctrl_dev()
846 	rpdev_ctrl = &vch->rpdev;  in rpmsg_virtio_add_ctrl_dev()
847 	rpdev_ctrl->ops = &virtio_rpmsg_ops;  in rpmsg_virtio_add_ctrl_dev()
849 	rpdev_ctrl->dev.parent = &vrp->vdev->dev;  in rpmsg_virtio_add_ctrl_dev()
850 	rpdev_ctrl->dev.release = virtio_rpmsg_release_device;  in rpmsg_virtio_add_ctrl_dev()
851 	rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_virtio_add_ctrl_dev()
855 		/* vch will be free in virtio_rpmsg_release_device() */  in rpmsg_virtio_add_ctrl_dev()
866 	device_unregister(&rpdev_ctrl->dev);  in rpmsg_virtio_del_ctrl_dev()
886 		return -ENOMEM;  in rpmsg_probe()
888 	vrp->vdev = vdev;  in rpmsg_probe()
890 	idr_init(&vrp->endpoints);  in rpmsg_probe()
891 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
892 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
893 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
895 	/* We expect two virtqueues, rx and tx (and in this order) */  in rpmsg_probe()
900 	vrp->rvq = vqs[0];  in rpmsg_probe()
901 	vrp->svq = vqs[1];  in rpmsg_probe()
904 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
905 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
908 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
909 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
911 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
913 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
915 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
918 	bufs_va = dma_alloc_coherent(vdev->dev.parent,  in rpmsg_probe()
919 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
922 		err = -ENOMEM;  in rpmsg_probe()
926 	dev_dbg(&vdev->dev, "buffers: va %pK, dma %pad\n",  in rpmsg_probe()
927 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
930 	vrp->rbufs = bufs_va;  in rpmsg_probe()
933 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
936 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
938 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
940 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
942 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
947 	/* suppress "tx-complete" interrupts */  in rpmsg_probe()
948 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
950 	vdev->priv = vrp;  in rpmsg_probe()
962 			err = -ENOMEM;  in rpmsg_probe()
967 		vch->vrp = vrp;  in rpmsg_probe()
970 		rpdev_ns = &vch->rpdev;  in rpmsg_probe()
971 		rpdev_ns->ops = &virtio_rpmsg_ops;  in rpmsg_probe()
972 		rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_probe()
974 		rpdev_ns->dev.parent = &vrp->vdev->dev;  in rpmsg_probe()
975 		rpdev_ns->dev.release = virtio_rpmsg_release_device;  in rpmsg_probe()
979 			/* vch will be free in virtio_rpmsg_release_device() */  in rpmsg_probe()
984 	 * Prepare to kick but don't notify yet - we can't do this before  in rpmsg_probe()
987 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
998 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
1000 	dev_info(&vdev->dev, "rpmsg host is online\n");  in rpmsg_probe()
1007 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_probe()
1008 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
1010 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
1025 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()
1026 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1031 	ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device);  in rpmsg_remove()
1033 		dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret);  in rpmsg_remove()
1035 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1037 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1039 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_remove()
1040 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1082 MODULE_DESCRIPTION("Virtio-based remote processor messaging bus");