Lines Matching +full:free +full:- +full:flowing
1 // SPDX-License-Identifier: GPL-2.0
6 * Copyright (C) 2020-2022 OpenSynergy.
7 * Copyright (C) 2021-2024 ARM Ltd.
13 * The scmi-virtio transport implements a driver for the virtio SCMI device.
40 * struct scmi_vio_channel - Transport channel information
49 * @pending_cmds_list: List of pre-fetched commands queueud for later processing
60 /* lock to protect access to the free list. */
87 * struct scmi_vio_msg - Transport PDU information
122 spin_lock_irqsave(&vioch->lock, flags); in scmi_vio_channel_ready()
123 cinfo->transport_info = vioch; in scmi_vio_channel_ready()
125 vioch->cinfo = cinfo; in scmi_vio_channel_ready()
126 spin_unlock_irqrestore(&vioch->lock, flags); in scmi_vio_channel_ready()
128 refcount_set(&vioch->users, 1); in scmi_vio_channel_ready()
133 return refcount_inc_not_zero(&vioch->users); in scmi_vio_channel_acquire()
138 if (refcount_dec_and_test(&vioch->users)) { in scmi_vio_channel_release()
141 spin_lock_irqsave(&vioch->lock, flags); in scmi_vio_channel_release()
142 if (vioch->shutdown_done) { in scmi_vio_channel_release()
143 vioch->cinfo = NULL; in scmi_vio_channel_release()
144 complete(vioch->shutdown_done); in scmi_vio_channel_release()
146 spin_unlock_irqrestore(&vioch->lock, flags); in scmi_vio_channel_release()
159 spin_lock_irqsave(&vioch->lock, flags); in scmi_vio_channel_cleanup_sync()
160 if (!vioch->cinfo || vioch->shutdown_done) { in scmi_vio_channel_cleanup_sync()
161 spin_unlock_irqrestore(&vioch->lock, flags); in scmi_vio_channel_cleanup_sync()
165 vioch->shutdown_done = &vioch_shutdown_done; in scmi_vio_channel_cleanup_sync()
166 if (!vioch->is_rx && vioch->deferred_tx_wq) in scmi_vio_channel_cleanup_sync()
168 vioch->deferred_tx_wq = NULL; in scmi_vio_channel_cleanup_sync()
169 spin_unlock_irqrestore(&vioch->lock, flags); in scmi_vio_channel_cleanup_sync()
174 wait_for_completion(vioch->shutdown_done); in scmi_vio_channel_cleanup_sync()
184 spin_lock_irqsave(&vioch->free_lock, flags); in scmi_virtio_get_free_msg()
185 if (list_empty(&vioch->free_list)) { in scmi_virtio_get_free_msg()
186 spin_unlock_irqrestore(&vioch->free_lock, flags); in scmi_virtio_get_free_msg()
190 msg = list_first_entry(&vioch->free_list, typeof(*msg), list); in scmi_virtio_get_free_msg()
191 list_del_init(&msg->list); in scmi_virtio_get_free_msg()
192 spin_unlock_irqrestore(&vioch->free_lock, flags); in scmi_virtio_get_free_msg()
195 msg->poll_status = VIO_MSG_NOT_POLLED; in scmi_virtio_get_free_msg()
196 refcount_set(&msg->users, 1); in scmi_virtio_get_free_msg()
203 return refcount_inc_not_zero(&msg->users); in scmi_vio_msg_acquire()
212 ret = refcount_dec_and_test(&msg->users); in scmi_vio_msg_release()
216 spin_lock_irqsave(&vioch->free_lock, flags); in scmi_vio_msg_release()
217 list_add_tail(&msg->list, &vioch->free_list); in scmi_vio_msg_release()
218 spin_unlock_irqrestore(&vioch->free_lock, flags); in scmi_vio_msg_release()
235 struct device *dev = &vioch->vqueue->vdev->dev; in scmi_vio_feed_vq_rx()
237 sg_init_one(&sg_in, msg->input, VIRTIO_SCMI_MAX_PDU_SIZE); in scmi_vio_feed_vq_rx()
239 spin_lock_irqsave(&vioch->lock, flags); in scmi_vio_feed_vq_rx()
241 rc = virtqueue_add_inbuf(vioch->vqueue, &sg_in, 1, msg, GFP_ATOMIC); in scmi_vio_feed_vq_rx()
245 virtqueue_kick(vioch->vqueue); in scmi_vio_feed_vq_rx()
247 spin_unlock_irqrestore(&vioch->lock, flags); in scmi_vio_feed_vq_rx()
254 * vioch->lock MUST NOT have been already acquired.
259 if (vioch->is_rx) in scmi_finalize_message()
273 if (WARN_ON_ONCE(!vqueue->vdev->priv)) in scmi_vio_complete_cb()
275 vioch = &((struct scmi_vio_channel *)vqueue->vdev->priv)[vqueue->index]; in scmi_vio_complete_cb()
281 spin_lock_irqsave(&vioch->lock, flags); in scmi_vio_complete_cb()
290 spin_unlock_irqrestore(&vioch->lock, flags); in scmi_vio_complete_cb()
296 spin_unlock_irqrestore(&vioch->lock, flags); in scmi_vio_complete_cb()
299 msg->rx_len = length; in scmi_vio_complete_cb()
300 core->rx_callback(vioch->cinfo, in scmi_vio_complete_cb()
301 core->msg->read_header(msg->input), in scmi_vio_complete_cb()
330 * Process pre-fetched messages: these could be non-polled messages or in scmi_vio_deferred_tx_worker()
331 * late timed-out replies to polled messages dequeued by chance while in scmi_vio_deferred_tx_worker()
333 * the valid non-expired messages and anyway finally free all of them. in scmi_vio_deferred_tx_worker()
335 spin_lock_irqsave(&vioch->pending_lock, flags); in scmi_vio_deferred_tx_worker()
337 /* Scan the list of possibly pre-fetched messages during polling. */ in scmi_vio_deferred_tx_worker()
338 list_for_each_entry_safe(msg, tmp, &vioch->pending_cmds_list, list) { in scmi_vio_deferred_tx_worker()
339 list_del(&msg->list); in scmi_vio_deferred_tx_worker()
345 if (msg->poll_status == VIO_MSG_NOT_POLLED) in scmi_vio_deferred_tx_worker()
346 core->rx_callback(vioch->cinfo, in scmi_vio_deferred_tx_worker()
347 core->msg->read_header(msg->input), in scmi_vio_deferred_tx_worker()
350 /* Free the processed message once done */ in scmi_vio_deferred_tx_worker()
354 spin_unlock_irqrestore(&vioch->pending_lock, flags); in scmi_vio_deferred_tx_worker()
357 scmi_vio_complete_cb(vioch->vqueue); in scmi_vio_deferred_tx_worker()
369 struct scmi_vio_channel *vioch = base_cinfo->transport_info; in virtio_get_max_msg()
371 return vioch->max_msg; in virtio_get_max_msg()
381 channels = (struct scmi_vio_channel *)scmi_vdev->priv; in virtio_chan_available()
395 return vioch && !vioch->cinfo; in virtio_chan_available()
411 return -EPROBE_DEFER; in virtio_chan_setup()
413 vioch = &((struct scmi_vio_channel *)scmi_vdev->priv)[index]; in virtio_chan_setup()
416 if (tx && !vioch->deferred_tx_wq) { in virtio_chan_setup()
419 vioch->deferred_tx_wq = in virtio_chan_setup()
420 alloc_workqueue(dev_name(&scmi_vdev->dev), in virtio_chan_setup()
423 if (!vioch->deferred_tx_wq) in virtio_chan_setup()
424 return -ENOMEM; in virtio_chan_setup()
427 vioch->deferred_tx_wq); in virtio_chan_setup()
431 INIT_WORK(&vioch->deferred_tx_work, in virtio_chan_setup()
435 for (i = 0; i < vioch->max_msg; i++) { in virtio_chan_setup()
440 return -ENOMEM; in virtio_chan_setup()
443 msg->request = devm_kzalloc(dev, in virtio_chan_setup()
446 if (!msg->request) in virtio_chan_setup()
447 return -ENOMEM; in virtio_chan_setup()
448 spin_lock_init(&msg->poll_lock); in virtio_chan_setup()
449 refcount_set(&msg->users, 1); in virtio_chan_setup()
452 msg->input = devm_kzalloc(dev, VIRTIO_SCMI_MAX_PDU_SIZE, in virtio_chan_setup()
454 if (!msg->input) in virtio_chan_setup()
455 return -ENOMEM; in virtio_chan_setup()
468 struct scmi_vio_channel *vioch = cinfo->transport_info; in virtio_chan_free()
471 * Break device to inhibit further traffic flowing while shutting down in virtio_chan_free()
472 * the channels: doing it later holding vioch->lock creates unsafe in virtio_chan_free()
475 virtio_break_device(vioch->vqueue->vdev); in virtio_chan_free()
484 struct scmi_vio_channel *vioch = cinfo->transport_info; in virtio_send_message()
493 return -EINVAL; in virtio_send_message()
498 return -EBUSY; in virtio_send_message()
501 core->msg->tx_prepare(msg->request, xfer); in virtio_send_message()
503 sg_init_one(&sg_out, msg->request, core->msg->command_size(xfer)); in virtio_send_message()
504 sg_init_one(&sg_in, msg->input, core->msg->response_size(xfer)); in virtio_send_message()
506 spin_lock_irqsave(&vioch->lock, flags); in virtio_send_message()
510 * - retrieve last used index (will be used as polling reference) in virtio_send_message()
511 * - bind the polled message to the xfer via .priv in virtio_send_message()
512 * - grab an additional msg refcount for the poll-path in virtio_send_message()
514 if (xfer->hdr.poll_completion) { in virtio_send_message()
515 msg->poll_idx = virtqueue_enable_cb_prepare(vioch->vqueue); in virtio_send_message()
517 msg->poll_status = VIO_MSG_POLLING; in virtio_send_message()
520 smp_store_mb(xfer->priv, msg); in virtio_send_message()
523 rc = virtqueue_add_sgs(vioch->vqueue, sgs, 1, 1, msg, GFP_ATOMIC); in virtio_send_message()
525 dev_err(vioch->cinfo->dev, in virtio_send_message()
528 virtqueue_kick(vioch->vqueue); in virtio_send_message()
530 spin_unlock_irqrestore(&vioch->lock, flags); in virtio_send_message()
533 /* Ensure order between xfer->priv clear and vq feeding */ in virtio_send_message()
534 smp_store_mb(xfer->priv, NULL); in virtio_send_message()
535 if (xfer->hdr.poll_completion) in virtio_send_message()
548 struct scmi_vio_msg *msg = xfer->priv; in virtio_fetch_response()
551 core->msg->fetch_response(msg->input, msg->rx_len, xfer); in virtio_fetch_response()
557 struct scmi_vio_msg *msg = xfer->priv; in virtio_fetch_notification()
560 core->msg->fetch_notification(msg->input, msg->rx_len, in virtio_fetch_notification()
565 * virtio_mark_txdone - Mark transmission done
567 * Free only completed polling transfer messages.
570 * outstanding but timed-out messages by forcibly re-adding them to the
571 * free-list inside the TX code path; we instead let IRQ/RX callbacks, or the
575 * This approach was deemed preferable since those pending timed-out buffers are
583 * For this same reason, here, we take care to free only the polled messages
585 * IRQ path - the initial scmi_vio_msg_release() takes care of this) and also
586 * any timed-out polled message if that indeed appears to have been at least
591 * Possible late replies to timed-out polled messages will be eventually freed
603 struct scmi_vio_channel *vioch = cinfo->transport_info; in virtio_mark_txdone()
604 struct scmi_vio_msg *msg = xfer->priv; in virtio_mark_txdone()
610 smp_store_mb(xfer->priv, NULL); in virtio_mark_txdone()
613 if (!xfer->hdr.poll_completion || scmi_vio_msg_release(vioch, msg)) { in virtio_mark_txdone()
618 spin_lock_irqsave(&msg->poll_lock, flags); in virtio_mark_txdone()
619 /* Do not free timedout polled messages only if still inflight */ in virtio_mark_txdone()
620 if (ret != -ETIMEDOUT || msg->poll_status == VIO_MSG_POLL_DONE) in virtio_mark_txdone()
622 else if (msg->poll_status == VIO_MSG_POLLING) in virtio_mark_txdone()
623 msg->poll_status = VIO_MSG_POLL_TIMEOUT; in virtio_mark_txdone()
624 spin_unlock_irqrestore(&msg->poll_lock, flags); in virtio_mark_txdone()
630 * virtio_poll_done - Provide polling support for VirtIO transport
642 * we were poll-waiting for: if that is the case such early fetched buffers are
646 * So, basically, once something new is spotted we proceed to de-queue all the
652 * busy-waiting helper.
654 * Finally, we delegate to the deferred worker also the final free of any timed
657 * Note that, since we do NOT have per-message suppress notification mechanism,
672 struct scmi_vio_msg *next_msg, *msg = xfer->priv; in virtio_poll_done()
673 struct scmi_vio_channel *vioch = cinfo->transport_info; in virtio_poll_done()
693 if (msg->poll_status == VIO_MSG_POLL_DONE) in virtio_poll_done()
700 pending = virtqueue_poll(vioch->vqueue, msg->poll_idx); in virtio_poll_done()
706 spin_lock_irqsave(&vioch->lock, flags); in virtio_poll_done()
707 virtqueue_disable_cb(vioch->vqueue); in virtio_poll_done()
710 * Process all new messages till the polled-for message is found OR in virtio_poll_done()
713 while ((next_msg = virtqueue_get_buf(vioch->vqueue, &length))) { in virtio_poll_done()
720 spin_lock(&next_msg->poll_lock); in virtio_poll_done()
721 if (next_msg->poll_status == VIO_MSG_POLLING) { in virtio_poll_done()
722 next_msg->poll_status = VIO_MSG_POLL_DONE; in virtio_poll_done()
725 spin_unlock(&next_msg->poll_lock); in virtio_poll_done()
727 next_msg->rx_len = length; in virtio_poll_done()
738 * Enqueue for later processing any non-polled message and any in virtio_poll_done()
739 * timed-out polled one that we happen to have dequeued. in virtio_poll_done()
741 spin_lock(&next_msg->poll_lock); in virtio_poll_done()
742 if (next_msg->poll_status == VIO_MSG_NOT_POLLED || in virtio_poll_done()
743 next_msg->poll_status == VIO_MSG_POLL_TIMEOUT) { in virtio_poll_done()
744 spin_unlock(&next_msg->poll_lock); in virtio_poll_done()
747 spin_lock(&vioch->pending_lock); in virtio_poll_done()
748 list_add_tail(&next_msg->list, in virtio_poll_done()
749 &vioch->pending_cmds_list); in virtio_poll_done()
750 spin_unlock(&vioch->pending_lock); in virtio_poll_done()
752 spin_unlock(&next_msg->poll_lock); in virtio_poll_done()
765 pending = !virtqueue_enable_cb(vioch->vqueue); in virtio_poll_done()
767 msg->poll_idx = virtqueue_enable_cb_prepare(vioch->vqueue); in virtio_poll_done()
768 pending = virtqueue_poll(vioch->vqueue, msg->poll_idx); in virtio_poll_done()
771 if (vioch->deferred_tx_wq && (any_prefetched || pending)) in virtio_poll_done()
772 queue_work(vioch->deferred_tx_wq, &vioch->deferred_tx_work); in virtio_poll_done()
774 spin_unlock_irqrestore(&vioch->lock, flags); in virtio_poll_done()
795 /* for non-realtime virtio devices */
803 { .compatible = "arm,scmi-virtio" },
812 struct device *dev = &vdev->dev; in scmi_vio_probe()
824 return -EBUSY; in scmi_vio_probe()
832 return -ENOMEM; in scmi_vio_probe()
868 vdev->priv = channels; in scmi_vio_probe()
875 vdev->priv = NULL; in scmi_vio_probe()
876 vdev->config->del_vqs(vdev); in scmi_vio_probe()
898 vdev->config->del_vqs(vdev); in scmi_vio_remove()
907 dev_err(&vdev->dev, in scmi_vio_validate()
909 return -EINVAL; in scmi_vio_validate()
925 .driver.name = "scmi-virtio",