1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * virtio transport for vsock
4 *
5 * Copyright (C) 2013-2015 Red Hat, Inc.
6 * Author: Asias He <asias@redhat.com>
7 * Stefan Hajnoczi <stefanha@redhat.com>
8 *
9 * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s
10 * early virtio-vsock proof-of-concept bits.
11 */
12 #include <linux/spinlock.h>
13 #include <linux/module.h>
14 #include <linux/list.h>
15 #include <linux/atomic.h>
16 #include <linux/virtio.h>
17 #include <linux/virtio_ids.h>
18 #include <linux/virtio_config.h>
19 #include <linux/virtio_vsock.h>
20 #include <net/sock.h>
21 #include <linux/mutex.h>
22 #include <net/af_vsock.h>
23
24 static struct workqueue_struct *virtio_vsock_workqueue;
25 static struct virtio_vsock __rcu *the_virtio_vsock;
26 static DEFINE_MUTEX(the_virtio_vsock_mutex); /* protects the_virtio_vsock */
27 static struct virtio_transport virtio_transport; /* forward declaration */
28
29 struct virtio_vsock {
30 struct virtio_device *vdev;
31 struct virtqueue *vqs[VSOCK_VQ_MAX];
32
33 /* Virtqueue processing is deferred to a workqueue */
34 struct work_struct tx_work;
35 struct work_struct rx_work;
36 struct work_struct event_work;
37
38 /* The following fields are protected by tx_lock. vqs[VSOCK_VQ_TX]
39 * must be accessed with tx_lock held.
40 */
41 struct mutex tx_lock;
42 bool tx_run;
43
44 struct work_struct send_pkt_work;
45 struct sk_buff_head send_pkt_queue;
46
47 atomic_t queued_replies;
48
49 /* The following fields are protected by rx_lock. vqs[VSOCK_VQ_RX]
50 * must be accessed with rx_lock held.
51 */
52 struct mutex rx_lock;
53 bool rx_run;
54 int rx_buf_nr;
55 int rx_buf_max_nr;
56
57 /* The following fields are protected by event_lock.
58 * vqs[VSOCK_VQ_EVENT] must be accessed with event_lock held.
59 */
60 struct mutex event_lock;
61 bool event_run;
62 struct virtio_vsock_event event_list[8];
63
64 u32 guest_cid;
65 bool seqpacket_allow;
66
67 /* These fields are used only in tx path in function
68 * 'virtio_transport_send_pkt_work()', so to save
69 * stack space in it, place both of them here. Each
70 * pointer from 'out_sgs' points to the corresponding
71 * element in 'out_bufs' - this is initialized in
72 * 'virtio_vsock_probe()'. Both fields are protected
73 * by 'tx_lock'. +1 is needed for packet header.
74 */
75 struct scatterlist *out_sgs[MAX_SKB_FRAGS + 1];
76 struct scatterlist out_bufs[MAX_SKB_FRAGS + 1];
77 };
78
virtio_transport_get_local_cid(void)79 static u32 virtio_transport_get_local_cid(void)
80 {
81 struct virtio_vsock *vsock;
82 u32 ret;
83
84 rcu_read_lock();
85 vsock = rcu_dereference(the_virtio_vsock);
86 if (!vsock) {
87 ret = VMADDR_CID_ANY;
88 goto out_rcu;
89 }
90
91 ret = vsock->guest_cid;
92 out_rcu:
93 rcu_read_unlock();
94 return ret;
95 }
96
97 /* Caller need to hold vsock->tx_lock on vq */
virtio_transport_send_skb(struct sk_buff * skb,struct virtqueue * vq,struct virtio_vsock * vsock,gfp_t gfp)98 static int virtio_transport_send_skb(struct sk_buff *skb, struct virtqueue *vq,
99 struct virtio_vsock *vsock, gfp_t gfp)
100 {
101 int ret, in_sg = 0, out_sg = 0;
102 struct scatterlist **sgs;
103
104 sgs = vsock->out_sgs;
105 sg_init_one(sgs[out_sg], virtio_vsock_hdr(skb),
106 sizeof(*virtio_vsock_hdr(skb)));
107 out_sg++;
108
109 if (!skb_is_nonlinear(skb)) {
110 if (skb->len > 0) {
111 sg_init_one(sgs[out_sg], skb->data, skb->len);
112 out_sg++;
113 }
114 } else {
115 struct skb_shared_info *si;
116 int i;
117
118 /* If skb is nonlinear, then its buffer must contain
119 * only header and nothing more. Data is stored in
120 * the fragged part.
121 */
122 WARN_ON_ONCE(skb_headroom(skb) != sizeof(*virtio_vsock_hdr(skb)));
123
124 si = skb_shinfo(skb);
125
126 for (i = 0; i < si->nr_frags; i++) {
127 skb_frag_t *skb_frag = &si->frags[i];
128 void *va;
129
130 /* We will use 'page_to_virt()' for the userspace page
131 * here, because virtio or dma-mapping layers will call
132 * 'virt_to_phys()' later to fill the buffer descriptor.
133 * We don't touch memory at "virtual" address of this page.
134 */
135 va = page_to_virt(skb_frag_page(skb_frag));
136 sg_init_one(sgs[out_sg],
137 va + skb_frag_off(skb_frag),
138 skb_frag_size(skb_frag));
139 out_sg++;
140 }
141 }
142
143 ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, skb, gfp);
144 /* Usually this means that there is no more space available in
145 * the vq
146 */
147 if (ret < 0)
148 return ret;
149
150 virtio_transport_deliver_tap_pkt(skb);
151 return 0;
152 }
153
154 static void
virtio_transport_send_pkt_work(struct work_struct * work)155 virtio_transport_send_pkt_work(struct work_struct *work)
156 {
157 struct virtio_vsock *vsock =
158 container_of(work, struct virtio_vsock, send_pkt_work);
159 struct virtqueue *vq;
160 bool added = false;
161 bool restart_rx = false;
162
163 mutex_lock(&vsock->tx_lock);
164
165 if (!vsock->tx_run)
166 goto out;
167
168 vq = vsock->vqs[VSOCK_VQ_TX];
169
170 for (;;) {
171 struct sk_buff *skb;
172 bool reply;
173 int ret;
174
175 skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);
176 if (!skb)
177 break;
178
179 reply = virtio_vsock_skb_reply(skb);
180
181 ret = virtio_transport_send_skb(skb, vq, vsock, GFP_KERNEL);
182 if (ret < 0) {
183 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
184 break;
185 }
186
187 if (reply) {
188 struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];
189 int val;
190
191 val = atomic_dec_return(&vsock->queued_replies);
192
193 /* Do we now have resources to resume rx processing? */
194 if (val + 1 == virtqueue_get_vring_size(rx_vq))
195 restart_rx = true;
196 }
197
198 added = true;
199 }
200
201 if (added)
202 virtqueue_kick(vq);
203
204 out:
205 mutex_unlock(&vsock->tx_lock);
206
207 if (restart_rx)
208 queue_work(virtio_vsock_workqueue, &vsock->rx_work);
209 }
210
211 /* Caller need to hold RCU for vsock.
212 * Returns 0 if the packet is successfully put on the vq.
213 */
virtio_transport_send_skb_fast_path(struct virtio_vsock * vsock,struct sk_buff * skb)214 static int virtio_transport_send_skb_fast_path(struct virtio_vsock *vsock, struct sk_buff *skb)
215 {
216 struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX];
217 int ret;
218
219 /* Inside RCU, can't sleep! */
220 ret = mutex_trylock(&vsock->tx_lock);
221 if (unlikely(ret == 0))
222 return -EBUSY;
223
224 ret = virtio_transport_send_skb(skb, vq, vsock, GFP_ATOMIC);
225 if (ret == 0)
226 virtqueue_kick(vq);
227
228 mutex_unlock(&vsock->tx_lock);
229
230 return ret;
231 }
232
233 static int
virtio_transport_send_pkt(struct sk_buff * skb)234 virtio_transport_send_pkt(struct sk_buff *skb)
235 {
236 struct virtio_vsock_hdr *hdr;
237 struct virtio_vsock *vsock;
238 int len = skb->len;
239
240 hdr = virtio_vsock_hdr(skb);
241
242 rcu_read_lock();
243 vsock = rcu_dereference(the_virtio_vsock);
244 if (!vsock) {
245 kfree_skb(skb);
246 len = -ENODEV;
247 goto out_rcu;
248 }
249
250 if (le64_to_cpu(hdr->dst_cid) == vsock->guest_cid) {
251 kfree_skb(skb);
252 len = -ENODEV;
253 goto out_rcu;
254 }
255
256 /* If send_pkt_queue is empty, we can safely bypass this queue
257 * because packet order is maintained and (try) to put the packet
258 * on the virtqueue using virtio_transport_send_skb_fast_path.
259 * If this fails we simply put the packet on the intermediate
260 * queue and schedule the worker.
261 */
262 if (!skb_queue_empty_lockless(&vsock->send_pkt_queue) ||
263 virtio_transport_send_skb_fast_path(vsock, skb)) {
264 if (virtio_vsock_skb_reply(skb))
265 atomic_inc(&vsock->queued_replies);
266
267 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);
268 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);
269 }
270
271 out_rcu:
272 rcu_read_unlock();
273 return len;
274 }
275
276 static int
virtio_transport_cancel_pkt(struct vsock_sock * vsk)277 virtio_transport_cancel_pkt(struct vsock_sock *vsk)
278 {
279 struct virtio_vsock *vsock;
280 int cnt = 0, ret;
281
282 rcu_read_lock();
283 vsock = rcu_dereference(the_virtio_vsock);
284 if (!vsock) {
285 ret = -ENODEV;
286 goto out_rcu;
287 }
288
289 cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue);
290
291 if (cnt) {
292 struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];
293 int new_cnt;
294
295 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
296 if (new_cnt + cnt >= virtqueue_get_vring_size(rx_vq) &&
297 new_cnt < virtqueue_get_vring_size(rx_vq))
298 queue_work(virtio_vsock_workqueue, &vsock->rx_work);
299 }
300
301 ret = 0;
302
303 out_rcu:
304 rcu_read_unlock();
305 return ret;
306 }
307
virtio_vsock_rx_fill(struct virtio_vsock * vsock)308 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)
309 {
310 int total_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM;
311 struct scatterlist pkt, *p;
312 struct virtqueue *vq;
313 struct sk_buff *skb;
314 int ret;
315
316 vq = vsock->vqs[VSOCK_VQ_RX];
317
318 do {
319 skb = virtio_vsock_alloc_skb(total_len, GFP_KERNEL);
320 if (!skb)
321 break;
322
323 memset(skb->head, 0, VIRTIO_VSOCK_SKB_HEADROOM);
324 sg_init_one(&pkt, virtio_vsock_hdr(skb), total_len);
325 p = &pkt;
326 ret = virtqueue_add_sgs(vq, &p, 0, 1, skb, GFP_KERNEL);
327 if (ret < 0) {
328 kfree_skb(skb);
329 break;
330 }
331
332 vsock->rx_buf_nr++;
333 } while (vq->num_free);
334 if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)
335 vsock->rx_buf_max_nr = vsock->rx_buf_nr;
336 virtqueue_kick(vq);
337 }
338
virtio_transport_tx_work(struct work_struct * work)339 static void virtio_transport_tx_work(struct work_struct *work)
340 {
341 struct virtio_vsock *vsock =
342 container_of(work, struct virtio_vsock, tx_work);
343 struct virtqueue *vq;
344 bool added = false;
345
346 vq = vsock->vqs[VSOCK_VQ_TX];
347 mutex_lock(&vsock->tx_lock);
348
349 if (!vsock->tx_run)
350 goto out;
351
352 do {
353 struct sk_buff *skb;
354 unsigned int len;
355
356 virtqueue_disable_cb(vq);
357 while ((skb = virtqueue_get_buf(vq, &len)) != NULL) {
358 virtio_transport_consume_skb_sent(skb, true);
359 added = true;
360 }
361 } while (!virtqueue_enable_cb(vq));
362
363 out:
364 mutex_unlock(&vsock->tx_lock);
365
366 if (added)
367 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);
368 }
369
370 /* Is there space left for replies to rx packets? */
virtio_transport_more_replies(struct virtio_vsock * vsock)371 static bool virtio_transport_more_replies(struct virtio_vsock *vsock)
372 {
373 struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];
374 int val;
375
376 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
377 val = atomic_read(&vsock->queued_replies);
378
379 return val < virtqueue_get_vring_size(vq);
380 }
381
382 /* event_lock must be held */
virtio_vsock_event_fill_one(struct virtio_vsock * vsock,struct virtio_vsock_event * event)383 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,
384 struct virtio_vsock_event *event)
385 {
386 struct scatterlist sg;
387 struct virtqueue *vq;
388
389 vq = vsock->vqs[VSOCK_VQ_EVENT];
390
391 sg_init_one(&sg, event, sizeof(*event));
392
393 return virtqueue_add_inbuf(vq, &sg, 1, event, GFP_KERNEL);
394 }
395
396 /* event_lock must be held */
virtio_vsock_event_fill(struct virtio_vsock * vsock)397 static void virtio_vsock_event_fill(struct virtio_vsock *vsock)
398 {
399 size_t i;
400
401 for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {
402 struct virtio_vsock_event *event = &vsock->event_list[i];
403
404 virtio_vsock_event_fill_one(vsock, event);
405 }
406
407 virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);
408 }
409
virtio_vsock_reset_sock(struct sock * sk)410 static void virtio_vsock_reset_sock(struct sock *sk)
411 {
412 /* vmci_transport.c doesn't take sk_lock here either. At least we're
413 * under vsock_table_lock so the sock cannot disappear while we're
414 * executing.
415 */
416
417 sk->sk_state = TCP_CLOSE;
418 sk->sk_err = ECONNRESET;
419 sk_error_report(sk);
420 }
421
virtio_vsock_update_guest_cid(struct virtio_vsock * vsock)422 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)
423 {
424 struct virtio_device *vdev = vsock->vdev;
425 __le64 guest_cid;
426
427 vdev->config->get(vdev, offsetof(struct virtio_vsock_config, guest_cid),
428 &guest_cid, sizeof(guest_cid));
429 vsock->guest_cid = le64_to_cpu(guest_cid);
430 }
431
432 /* event_lock must be held */
virtio_vsock_event_handle(struct virtio_vsock * vsock,struct virtio_vsock_event * event)433 static void virtio_vsock_event_handle(struct virtio_vsock *vsock,
434 struct virtio_vsock_event *event)
435 {
436 switch (le32_to_cpu(event->id)) {
437 case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET:
438 virtio_vsock_update_guest_cid(vsock);
439 vsock_for_each_connected_socket(&virtio_transport.transport,
440 virtio_vsock_reset_sock);
441 break;
442 }
443 }
444
virtio_transport_event_work(struct work_struct * work)445 static void virtio_transport_event_work(struct work_struct *work)
446 {
447 struct virtio_vsock *vsock =
448 container_of(work, struct virtio_vsock, event_work);
449 struct virtqueue *vq;
450
451 vq = vsock->vqs[VSOCK_VQ_EVENT];
452
453 mutex_lock(&vsock->event_lock);
454
455 if (!vsock->event_run)
456 goto out;
457
458 do {
459 struct virtio_vsock_event *event;
460 unsigned int len;
461
462 virtqueue_disable_cb(vq);
463 while ((event = virtqueue_get_buf(vq, &len)) != NULL) {
464 if (len == sizeof(*event))
465 virtio_vsock_event_handle(vsock, event);
466
467 virtio_vsock_event_fill_one(vsock, event);
468 }
469 } while (!virtqueue_enable_cb(vq));
470
471 virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);
472 out:
473 mutex_unlock(&vsock->event_lock);
474 }
475
virtio_vsock_event_done(struct virtqueue * vq)476 static void virtio_vsock_event_done(struct virtqueue *vq)
477 {
478 struct virtio_vsock *vsock = vq->vdev->priv;
479
480 if (!vsock)
481 return;
482 queue_work(virtio_vsock_workqueue, &vsock->event_work);
483 }
484
virtio_vsock_tx_done(struct virtqueue * vq)485 static void virtio_vsock_tx_done(struct virtqueue *vq)
486 {
487 struct virtio_vsock *vsock = vq->vdev->priv;
488
489 if (!vsock)
490 return;
491 queue_work(virtio_vsock_workqueue, &vsock->tx_work);
492 }
493
virtio_vsock_rx_done(struct virtqueue * vq)494 static void virtio_vsock_rx_done(struct virtqueue *vq)
495 {
496 struct virtio_vsock *vsock = vq->vdev->priv;
497
498 if (!vsock)
499 return;
500 queue_work(virtio_vsock_workqueue, &vsock->rx_work);
501 }
502
virtio_transport_can_msgzerocopy(int bufs_num)503 static bool virtio_transport_can_msgzerocopy(int bufs_num)
504 {
505 struct virtio_vsock *vsock;
506 bool res = false;
507
508 rcu_read_lock();
509
510 vsock = rcu_dereference(the_virtio_vsock);
511 if (vsock) {
512 struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX];
513
514 /* Check that tx queue is large enough to keep whole
515 * data to send. This is needed, because when there is
516 * not enough free space in the queue, current skb to
517 * send will be reinserted to the head of tx list of
518 * the socket to retry transmission later, so if skb
519 * is bigger than whole queue, it will be reinserted
520 * again and again, thus blocking other skbs to be sent.
521 * Each page of the user provided buffer will be added
522 * as a single buffer to the tx virtqueue, so compare
523 * number of pages against maximum capacity of the queue.
524 */
525 if (bufs_num <= vq->num_max)
526 res = true;
527 }
528
529 rcu_read_unlock();
530
531 return res;
532 }
533
virtio_transport_msgzerocopy_allow(void)534 static bool virtio_transport_msgzerocopy_allow(void)
535 {
536 return true;
537 }
538
539 static bool virtio_transport_seqpacket_allow(u32 remote_cid);
540
541 static struct virtio_transport virtio_transport = {
542 .transport = {
543 .module = THIS_MODULE,
544
545 .get_local_cid = virtio_transport_get_local_cid,
546
547 .init = virtio_transport_do_socket_init,
548 .destruct = virtio_transport_destruct,
549 .release = virtio_transport_release,
550 .connect = virtio_transport_connect,
551 .shutdown = virtio_transport_shutdown,
552 .cancel_pkt = virtio_transport_cancel_pkt,
553
554 .dgram_bind = virtio_transport_dgram_bind,
555 .dgram_dequeue = virtio_transport_dgram_dequeue,
556 .dgram_enqueue = virtio_transport_dgram_enqueue,
557 .dgram_allow = virtio_transport_dgram_allow,
558
559 .stream_dequeue = virtio_transport_stream_dequeue,
560 .stream_enqueue = virtio_transport_stream_enqueue,
561 .stream_has_data = virtio_transport_stream_has_data,
562 .stream_has_space = virtio_transport_stream_has_space,
563 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat,
564 .stream_is_active = virtio_transport_stream_is_active,
565 .stream_allow = virtio_transport_stream_allow,
566
567 .seqpacket_dequeue = virtio_transport_seqpacket_dequeue,
568 .seqpacket_enqueue = virtio_transport_seqpacket_enqueue,
569 .seqpacket_allow = virtio_transport_seqpacket_allow,
570 .seqpacket_has_data = virtio_transport_seqpacket_has_data,
571
572 .msgzerocopy_allow = virtio_transport_msgzerocopy_allow,
573
574 .notify_poll_in = virtio_transport_notify_poll_in,
575 .notify_poll_out = virtio_transport_notify_poll_out,
576 .notify_recv_init = virtio_transport_notify_recv_init,
577 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block,
578 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue,
579 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
580 .notify_send_init = virtio_transport_notify_send_init,
581 .notify_send_pre_block = virtio_transport_notify_send_pre_block,
582 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue,
583 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
584 .notify_buffer_size = virtio_transport_notify_buffer_size,
585 .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat,
586
587 .unsent_bytes = virtio_transport_unsent_bytes,
588
589 .read_skb = virtio_transport_read_skb,
590 },
591
592 .send_pkt = virtio_transport_send_pkt,
593 .can_msgzerocopy = virtio_transport_can_msgzerocopy,
594 };
595
virtio_transport_seqpacket_allow(u32 remote_cid)596 static bool virtio_transport_seqpacket_allow(u32 remote_cid)
597 {
598 struct virtio_vsock *vsock;
599 bool seqpacket_allow;
600
601 seqpacket_allow = false;
602 rcu_read_lock();
603 vsock = rcu_dereference(the_virtio_vsock);
604 if (vsock)
605 seqpacket_allow = vsock->seqpacket_allow;
606 rcu_read_unlock();
607
608 return seqpacket_allow;
609 }
610
virtio_transport_rx_work(struct work_struct * work)611 static void virtio_transport_rx_work(struct work_struct *work)
612 {
613 struct virtio_vsock *vsock =
614 container_of(work, struct virtio_vsock, rx_work);
615 struct virtqueue *vq;
616
617 vq = vsock->vqs[VSOCK_VQ_RX];
618
619 mutex_lock(&vsock->rx_lock);
620
621 if (!vsock->rx_run)
622 goto out;
623
624 do {
625 virtqueue_disable_cb(vq);
626 for (;;) {
627 struct sk_buff *skb;
628 unsigned int len;
629
630 if (!virtio_transport_more_replies(vsock)) {
631 /* Stop rx until the device processes already
632 * pending replies. Leave rx virtqueue
633 * callbacks disabled.
634 */
635 goto out;
636 }
637
638 skb = virtqueue_get_buf(vq, &len);
639 if (!skb)
640 break;
641
642 vsock->rx_buf_nr--;
643
644 /* Drop short/long packets */
645 if (unlikely(len < sizeof(struct virtio_vsock_hdr) ||
646 len > virtio_vsock_skb_len(skb))) {
647 kfree_skb(skb);
648 continue;
649 }
650
651 virtio_vsock_skb_rx_put(skb);
652 virtio_transport_deliver_tap_pkt(skb);
653 virtio_transport_recv_pkt(&virtio_transport, skb);
654 }
655 } while (!virtqueue_enable_cb(vq));
656
657 out:
658 if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)
659 virtio_vsock_rx_fill(vsock);
660 mutex_unlock(&vsock->rx_lock);
661 }
662
virtio_vsock_vqs_init(struct virtio_vsock * vsock)663 static int virtio_vsock_vqs_init(struct virtio_vsock *vsock)
664 {
665 struct virtio_device *vdev = vsock->vdev;
666 struct virtqueue_info vqs_info[] = {
667 { "rx", virtio_vsock_rx_done },
668 { "tx", virtio_vsock_tx_done },
669 { "event", virtio_vsock_event_done },
670 };
671 int ret;
672
673 ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, vqs_info, NULL);
674 if (ret < 0)
675 return ret;
676
677 virtio_vsock_update_guest_cid(vsock);
678
679 virtio_device_ready(vdev);
680
681 return 0;
682 }
683
virtio_vsock_vqs_start(struct virtio_vsock * vsock)684 static void virtio_vsock_vqs_start(struct virtio_vsock *vsock)
685 {
686 mutex_lock(&vsock->tx_lock);
687 vsock->tx_run = true;
688 mutex_unlock(&vsock->tx_lock);
689
690 mutex_lock(&vsock->rx_lock);
691 virtio_vsock_rx_fill(vsock);
692 vsock->rx_run = true;
693 mutex_unlock(&vsock->rx_lock);
694
695 mutex_lock(&vsock->event_lock);
696 virtio_vsock_event_fill(vsock);
697 vsock->event_run = true;
698 mutex_unlock(&vsock->event_lock);
699
700 /* virtio_transport_send_pkt() can queue packets once
701 * the_virtio_vsock is set, but they won't be processed until
702 * vsock->tx_run is set to true. We queue vsock->send_pkt_work
703 * when initialization finishes to send those packets queued
704 * earlier.
705 * We don't need to queue the other workers (rx, event) because
706 * as long as we don't fill the queues with empty buffers, the
707 * host can't send us any notification.
708 */
709 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);
710 }
711
virtio_vsock_vqs_del(struct virtio_vsock * vsock)712 static void virtio_vsock_vqs_del(struct virtio_vsock *vsock)
713 {
714 struct virtio_device *vdev = vsock->vdev;
715 struct sk_buff *skb;
716
717 /* Reset all connected sockets when the VQs disappear */
718 vsock_for_each_connected_socket(&virtio_transport.transport,
719 virtio_vsock_reset_sock);
720
721 /* Stop all work handlers to make sure no one is accessing the device,
722 * so we can safely call virtio_reset_device().
723 */
724 mutex_lock(&vsock->rx_lock);
725 vsock->rx_run = false;
726 mutex_unlock(&vsock->rx_lock);
727
728 mutex_lock(&vsock->tx_lock);
729 vsock->tx_run = false;
730 mutex_unlock(&vsock->tx_lock);
731
732 mutex_lock(&vsock->event_lock);
733 vsock->event_run = false;
734 mutex_unlock(&vsock->event_lock);
735
736 /* Flush all device writes and interrupts, device will not use any
737 * more buffers.
738 */
739 virtio_reset_device(vdev);
740
741 mutex_lock(&vsock->rx_lock);
742 while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))
743 kfree_skb(skb);
744 mutex_unlock(&vsock->rx_lock);
745
746 mutex_lock(&vsock->tx_lock);
747 while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))
748 kfree_skb(skb);
749 mutex_unlock(&vsock->tx_lock);
750
751 virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);
752
753 /* Delete virtqueues and flush outstanding callbacks if any */
754 vdev->config->del_vqs(vdev);
755 }
756
virtio_vsock_probe(struct virtio_device * vdev)757 static int virtio_vsock_probe(struct virtio_device *vdev)
758 {
759 struct virtio_vsock *vsock = NULL;
760 int ret;
761 int i;
762
763 ret = mutex_lock_interruptible(&the_virtio_vsock_mutex);
764 if (ret)
765 return ret;
766
767 /* Only one virtio-vsock device per guest is supported */
768 if (rcu_dereference_protected(the_virtio_vsock,
769 lockdep_is_held(&the_virtio_vsock_mutex))) {
770 ret = -EBUSY;
771 goto out;
772 }
773
774 vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);
775 if (!vsock) {
776 ret = -ENOMEM;
777 goto out;
778 }
779
780 vsock->vdev = vdev;
781
782 vsock->rx_buf_nr = 0;
783 vsock->rx_buf_max_nr = 0;
784 atomic_set(&vsock->queued_replies, 0);
785
786 mutex_init(&vsock->tx_lock);
787 mutex_init(&vsock->rx_lock);
788 mutex_init(&vsock->event_lock);
789 skb_queue_head_init(&vsock->send_pkt_queue);
790 INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);
791 INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);
792 INIT_WORK(&vsock->event_work, virtio_transport_event_work);
793 INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);
794
795 if (virtio_has_feature(vdev, VIRTIO_VSOCK_F_SEQPACKET))
796 vsock->seqpacket_allow = true;
797
798 vdev->priv = vsock;
799
800 ret = virtio_vsock_vqs_init(vsock);
801 if (ret < 0)
802 goto out;
803
804 for (i = 0; i < ARRAY_SIZE(vsock->out_sgs); i++)
805 vsock->out_sgs[i] = &vsock->out_bufs[i];
806
807 rcu_assign_pointer(the_virtio_vsock, vsock);
808 virtio_vsock_vqs_start(vsock);
809
810 mutex_unlock(&the_virtio_vsock_mutex);
811
812 return 0;
813
814 out:
815 kfree(vsock);
816 mutex_unlock(&the_virtio_vsock_mutex);
817 return ret;
818 }
819
virtio_vsock_remove(struct virtio_device * vdev)820 static void virtio_vsock_remove(struct virtio_device *vdev)
821 {
822 struct virtio_vsock *vsock = vdev->priv;
823
824 mutex_lock(&the_virtio_vsock_mutex);
825
826 vdev->priv = NULL;
827 rcu_assign_pointer(the_virtio_vsock, NULL);
828 synchronize_rcu();
829
830 virtio_vsock_vqs_del(vsock);
831
832 /* Other works can be queued before 'config->del_vqs()', so we flush
833 * all works before to free the vsock object to avoid use after free.
834 */
835 flush_work(&vsock->rx_work);
836 flush_work(&vsock->tx_work);
837 flush_work(&vsock->event_work);
838 flush_work(&vsock->send_pkt_work);
839
840 mutex_unlock(&the_virtio_vsock_mutex);
841
842 kfree(vsock);
843 }
844
845 #ifdef CONFIG_PM_SLEEP
virtio_vsock_freeze(struct virtio_device * vdev)846 static int virtio_vsock_freeze(struct virtio_device *vdev)
847 {
848 struct virtio_vsock *vsock = vdev->priv;
849
850 mutex_lock(&the_virtio_vsock_mutex);
851
852 rcu_assign_pointer(the_virtio_vsock, NULL);
853 synchronize_rcu();
854
855 virtio_vsock_vqs_del(vsock);
856
857 mutex_unlock(&the_virtio_vsock_mutex);
858
859 return 0;
860 }
861
virtio_vsock_restore(struct virtio_device * vdev)862 static int virtio_vsock_restore(struct virtio_device *vdev)
863 {
864 struct virtio_vsock *vsock = vdev->priv;
865 int ret;
866
867 mutex_lock(&the_virtio_vsock_mutex);
868
869 /* Only one virtio-vsock device per guest is supported */
870 if (rcu_dereference_protected(the_virtio_vsock,
871 lockdep_is_held(&the_virtio_vsock_mutex))) {
872 ret = -EBUSY;
873 goto out;
874 }
875
876 ret = virtio_vsock_vqs_init(vsock);
877 if (ret < 0)
878 goto out;
879
880 rcu_assign_pointer(the_virtio_vsock, vsock);
881 virtio_vsock_vqs_start(vsock);
882
883 out:
884 mutex_unlock(&the_virtio_vsock_mutex);
885 return ret;
886 }
887 #endif /* CONFIG_PM_SLEEP */
888
889 static struct virtio_device_id id_table[] = {
890 { VIRTIO_ID_VSOCK, VIRTIO_DEV_ANY_ID },
891 { 0 },
892 };
893
894 static unsigned int features[] = {
895 VIRTIO_VSOCK_F_SEQPACKET
896 };
897
898 static struct virtio_driver virtio_vsock_driver = {
899 .feature_table = features,
900 .feature_table_size = ARRAY_SIZE(features),
901 .driver.name = KBUILD_MODNAME,
902 .id_table = id_table,
903 .probe = virtio_vsock_probe,
904 .remove = virtio_vsock_remove,
905 #ifdef CONFIG_PM_SLEEP
906 .freeze = virtio_vsock_freeze,
907 .restore = virtio_vsock_restore,
908 #endif
909 };
910
virtio_vsock_init(void)911 static int __init virtio_vsock_init(void)
912 {
913 int ret;
914
915 virtio_vsock_workqueue = alloc_workqueue("virtio_vsock", 0, 0);
916 if (!virtio_vsock_workqueue)
917 return -ENOMEM;
918
919 ret = vsock_core_register(&virtio_transport.transport,
920 VSOCK_TRANSPORT_F_G2H);
921 if (ret)
922 goto out_wq;
923
924 ret = register_virtio_driver(&virtio_vsock_driver);
925 if (ret)
926 goto out_vci;
927
928 return 0;
929
930 out_vci:
931 vsock_core_unregister(&virtio_transport.transport);
932 out_wq:
933 destroy_workqueue(virtio_vsock_workqueue);
934 return ret;
935 }
936
virtio_vsock_exit(void)937 static void __exit virtio_vsock_exit(void)
938 {
939 unregister_virtio_driver(&virtio_vsock_driver);
940 vsock_core_unregister(&virtio_transport.transport);
941 destroy_workqueue(virtio_vsock_workqueue);
942 }
943
944 module_init(virtio_vsock_init);
945 module_exit(virtio_vsock_exit);
946 MODULE_LICENSE("GPL v2");
947 MODULE_AUTHOR("Asias He");
948 MODULE_DESCRIPTION("virtio transport for vsock");
949 MODULE_DEVICE_TABLE(virtio, id_table);
950