Lines Matching refs:tun
134 struct tun_struct __rcu *tun; member
160 struct tun_struct *tun; member
224 static void tun_flow_init(struct tun_struct *tun);
225 static void tun_flow_uninit(struct tun_struct *tun);
267 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, in tun_napi_init() argument
273 netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll); in tun_napi_init()
302 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) in tun_legacy_is_little_endian() argument
304 return tun->flags & TUN_VNET_BE ? false : in tun_legacy_is_little_endian()
308 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) in tun_get_vnet_be() argument
310 int be = !!(tun->flags & TUN_VNET_BE); in tun_get_vnet_be()
318 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) in tun_set_vnet_be() argument
326 tun->flags |= TUN_VNET_BE; in tun_set_vnet_be()
328 tun->flags &= ~TUN_VNET_BE; in tun_set_vnet_be()
333 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) in tun_legacy_is_little_endian() argument
338 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) in tun_get_vnet_be() argument
343 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) in tun_set_vnet_be() argument
349 static inline bool tun_is_little_endian(struct tun_struct *tun) in tun_is_little_endian() argument
351 return tun->flags & TUN_VNET_LE || in tun_is_little_endian()
352 tun_legacy_is_little_endian(tun); in tun_is_little_endian()
355 static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val) in tun16_to_cpu() argument
357 return __virtio16_to_cpu(tun_is_little_endian(tun), val); in tun16_to_cpu()
360 static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val) in cpu_to_tun16() argument
362 return __cpu_to_virtio16(tun_is_little_endian(tun), val); in cpu_to_tun16()
381 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun, in tun_flow_create() argument
388 netif_info(tun, tx_queued, tun->dev, in tun_flow_create()
395 e->tun = tun; in tun_flow_create()
397 ++tun->flow_count; in tun_flow_create()
402 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e) in tun_flow_delete() argument
404 netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n", in tun_flow_delete()
408 --tun->flow_count; in tun_flow_delete()
411 static void tun_flow_flush(struct tun_struct *tun) in tun_flow_flush() argument
415 spin_lock_bh(&tun->lock); in tun_flow_flush()
420 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) in tun_flow_flush()
421 tun_flow_delete(tun, e); in tun_flow_flush()
423 spin_unlock_bh(&tun->lock); in tun_flow_flush()
426 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index) in tun_flow_delete_by_queue() argument
430 spin_lock_bh(&tun->lock); in tun_flow_delete_by_queue()
435 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { in tun_flow_delete_by_queue()
437 tun_flow_delete(tun, e); in tun_flow_delete_by_queue()
440 spin_unlock_bh(&tun->lock); in tun_flow_delete_by_queue()
445 struct tun_struct *tun = from_timer(tun, t, flow_gc_timer); in tun_flow_cleanup() local
446 unsigned long delay = tun->ageing_time; in tun_flow_cleanup()
451 spin_lock(&tun->lock); in tun_flow_cleanup()
456 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { in tun_flow_cleanup()
461 tun_flow_delete(tun, e); in tun_flow_cleanup()
471 mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer)); in tun_flow_cleanup()
472 spin_unlock(&tun->lock); in tun_flow_cleanup()
475 static void tun_flow_update(struct tun_struct *tun, u32 rxhash, in tun_flow_update() argument
480 unsigned long delay = tun->ageing_time; in tun_flow_update()
483 head = &tun->flows[tun_hashfn(rxhash)]; in tun_flow_update()
496 spin_lock_bh(&tun->lock); in tun_flow_update()
498 tun->flow_count < MAX_TAP_FLOWS) in tun_flow_update()
499 tun_flow_create(tun, head, rxhash, queue_index); in tun_flow_update()
501 if (!timer_pending(&tun->flow_gc_timer)) in tun_flow_update()
502 mod_timer(&tun->flow_gc_timer, in tun_flow_update()
504 spin_unlock_bh(&tun->lock); in tun_flow_update()
525 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb) in tun_automq_select_queue() argument
530 numqueues = READ_ONCE(tun->numqueues); in tun_automq_select_queue()
533 e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq); in tun_automq_select_queue()
544 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb) in tun_ebpf_select_queue() argument
550 numqueues = READ_ONCE(tun->numqueues); in tun_ebpf_select_queue()
554 prog = rcu_dereference(tun->steering_prog); in tun_ebpf_select_queue()
564 struct tun_struct *tun = netdev_priv(dev); in tun_select_queue() local
568 if (rcu_dereference(tun->steering_prog)) in tun_select_queue()
569 ret = tun_ebpf_select_queue(tun, skb); in tun_select_queue()
571 ret = tun_automq_select_queue(tun, skb); in tun_select_queue()
577 static inline bool tun_not_capable(struct tun_struct *tun) in tun_not_capable() argument
580 struct net *net = dev_net(tun->dev); in tun_not_capable()
582 return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) || in tun_not_capable()
583 (gid_valid(tun->group) && !in_egroup_p(tun->group))) && in tun_not_capable()
587 static void tun_set_real_num_queues(struct tun_struct *tun) in tun_set_real_num_queues() argument
589 netif_set_real_num_tx_queues(tun->dev, tun->numqueues); in tun_set_real_num_queues()
590 netif_set_real_num_rx_queues(tun->dev, tun->numqueues); in tun_set_real_num_queues()
593 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile) in tun_disable_queue() argument
595 tfile->detached = tun; in tun_disable_queue()
596 list_add_tail(&tfile->next, &tun->disabled); in tun_disable_queue()
597 ++tun->numdisabled; in tun_disable_queue()
602 struct tun_struct *tun = tfile->detached; in tun_enable_queue() local
606 --tun->numdisabled; in tun_enable_queue()
607 return tun; in tun_enable_queue()
638 struct tun_struct *tun; in __tun_detach() local
640 tun = rtnl_dereference(tfile->tun); in __tun_detach()
642 if (tun && clean) { in __tun_detach()
648 if (tun && !tfile->detached) { in __tun_detach()
650 BUG_ON(index >= tun->numqueues); in __tun_detach()
652 rcu_assign_pointer(tun->tfiles[index], in __tun_detach()
653 tun->tfiles[tun->numqueues - 1]); in __tun_detach()
654 ntfile = rtnl_dereference(tun->tfiles[index]); in __tun_detach()
657 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], in __tun_detach()
660 --tun->numqueues; in __tun_detach()
662 RCU_INIT_POINTER(tfile->tun, NULL); in __tun_detach()
665 tun_disable_queue(tun, tfile); in __tun_detach()
670 tun_flow_delete_by_queue(tun, tun->numqueues + 1); in __tun_detach()
673 tun_set_real_num_queues(tun); in __tun_detach()
675 tun = tun_enable_queue(tfile); in __tun_detach()
680 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) { in __tun_detach()
681 netif_carrier_off(tun->dev); in __tun_detach()
683 if (!(tun->flags & IFF_PERSIST) && in __tun_detach()
684 tun->dev->reg_state == NETREG_REGISTERED) in __tun_detach()
685 unregister_netdevice(tun->dev); in __tun_detach()
687 if (tun) in __tun_detach()
695 struct tun_struct *tun; in tun_detach() local
699 tun = rtnl_dereference(tfile->tun); in tun_detach()
700 dev = tun ? tun->dev : NULL; in tun_detach()
712 struct tun_struct *tun = netdev_priv(dev); in tun_detach_all() local
714 int i, n = tun->numqueues; in tun_detach_all()
717 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_all()
722 RCU_INIT_POINTER(tfile->tun, NULL); in tun_detach_all()
723 --tun->numqueues; in tun_detach_all()
725 list_for_each_entry(tfile, &tun->disabled, next) { in tun_detach_all()
728 RCU_INIT_POINTER(tfile->tun, NULL); in tun_detach_all()
730 BUG_ON(tun->numqueues != 0); in tun_detach_all()
734 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_all()
741 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { in tun_detach_all()
748 BUG_ON(tun->numdisabled != 0); in tun_detach_all()
750 if (tun->flags & IFF_PERSIST) in tun_detach_all()
754 static int tun_attach(struct tun_struct *tun, struct file *file, in tun_attach() argument
759 struct net_device *dev = tun->dev; in tun_attach()
762 err = security_tun_dev_attach(tfile->socket.sk, tun->security); in tun_attach()
767 if (rtnl_dereference(tfile->tun) && !tfile->detached) in tun_attach()
771 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1) in tun_attach()
776 tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES) in tun_attach()
782 if (!skip_filter && (tun->filter_attached == true)) { in tun_attach()
784 err = sk_attach_filter(&tun->fprog, tfile->socket.sk); in tun_attach()
797 tfile->queue_index = tun->numqueues; in tun_attach()
809 tun->dev, tfile->queue_index, 0); in tun_attach()
826 tun_napi_init(tun, tfile, napi, napi_frags); in tun_attach()
829 if (rtnl_dereference(tun->xdp_prog)) in tun_attach()
841 rcu_assign_pointer(tfile->tun, tun); in tun_attach()
842 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile); in tun_attach()
843 tun->numqueues++; in tun_attach()
844 tun_set_real_num_queues(tun); in tun_attach()
851 struct tun_struct *tun; in tun_get() local
854 tun = rcu_dereference(tfile->tun); in tun_get()
855 if (tun) in tun_get()
856 dev_hold(tun->dev); in tun_get()
859 return tun; in tun_get()
862 static void tun_put(struct tun_struct *tun) in tun_put() argument
864 dev_put(tun->dev); in tun_put()
977 struct tun_struct *tun = netdev_priv(dev); in tun_net_init() local
978 struct ifreq *ifr = tun->ifr; in tun_net_init()
981 spin_lock_init(&tun->lock); in tun_net_init()
983 err = security_tun_dev_alloc_security(&tun->security); in tun_net_init()
987 tun_flow_init(tun); in tun_net_init()
999 tun->flags = (tun->flags & ~TUN_FEATURES) | in tun_net_init()
1002 INIT_LIST_HEAD(&tun->disabled); in tun_net_init()
1003 err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI, in tun_net_init()
1006 tun_flow_uninit(tun); in tun_net_init()
1007 security_tun_dev_free_security(tun->security); in tun_net_init()
1035 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb) in tun_automq_xmit() argument
1038 if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) { in tun_automq_xmit()
1046 e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash); in tun_automq_xmit()
1053 static unsigned int run_ebpf_filter(struct tun_struct *tun, in run_ebpf_filter() argument
1057 struct tun_prog *prog = rcu_dereference(tun->filter_prog); in run_ebpf_filter()
1068 struct tun_struct *tun = netdev_priv(dev); in tun_net_xmit() local
1076 tfile = rcu_dereference(tun->tfiles[txq]); in tun_net_xmit()
1084 if (!rcu_dereference(tun->steering_prog)) in tun_net_xmit()
1085 tun_automq_xmit(tun, skb); in tun_net_xmit()
1087 netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len); in tun_net_xmit()
1092 if (!check_filter(&tun->txflt, skb)) { in tun_net_xmit()
1103 len = run_ebpf_filter(tun, skb, len); in tun_net_xmit()
1165 struct tun_struct *tun = netdev_priv(dev); in tun_net_fix_features() local
1167 return (features & tun->set_features) | (features & ~TUN_USER_FEATURES); in tun_net_fix_features()
1172 struct tun_struct *tun = netdev_priv(dev); in tun_set_headroom() local
1177 tun->align = new_hr; in tun_set_headroom()
1183 struct tun_struct *tun = netdev_priv(dev); in tun_net_get_stats64() local
1188 (unsigned long)atomic_long_read(&tun->rx_frame_errors); in tun_net_get_stats64()
1194 struct tun_struct *tun = netdev_priv(dev); in tun_xdp_set() local
1199 old_prog = rtnl_dereference(tun->xdp_prog); in tun_xdp_set()
1200 rcu_assign_pointer(tun->xdp_prog, prog); in tun_xdp_set()
1204 for (i = 0; i < tun->numqueues; i++) { in tun_xdp_set()
1205 tfile = rtnl_dereference(tun->tfiles[i]); in tun_xdp_set()
1211 list_for_each_entry(tfile, &tun->disabled, next) { in tun_xdp_set()
1234 struct tun_struct *tun = netdev_priv(dev); in tun_net_change_carrier() local
1236 if (!tun->numqueues) in tun_net_change_carrier()
1270 struct tun_struct *tun = netdev_priv(dev); in tun_xdp_xmit() local
1282 numqueues = READ_ONCE(tun->numqueues); in tun_xdp_xmit()
1288 tfile = rcu_dereference(tun->tfiles[smp_processor_id() % in tun_xdp_xmit()
1348 static void tun_flow_init(struct tun_struct *tun) in tun_flow_init() argument
1353 INIT_HLIST_HEAD(&tun->flows[i]); in tun_flow_init()
1355 tun->ageing_time = TUN_FLOW_EXPIRE; in tun_flow_init()
1356 timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0); in tun_flow_init()
1357 mod_timer(&tun->flow_gc_timer, in tun_flow_init()
1358 round_jiffies_up(jiffies + tun->ageing_time)); in tun_flow_init()
1361 static void tun_flow_uninit(struct tun_struct *tun) in tun_flow_uninit() argument
1363 del_timer_sync(&tun->flow_gc_timer); in tun_flow_uninit()
1364 tun_flow_flush(tun); in tun_flow_uninit()
1373 struct tun_struct *tun = netdev_priv(dev); in tun_net_initialize() local
1375 switch (tun->flags & TUN_TYPE_MASK) { in tun_net_initialize()
1411 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile) in tun_sock_writeable() argument
1415 return (tun->dev->flags & IFF_UP) && sock_writeable(sk); in tun_sock_writeable()
1424 struct tun_struct *tun = tun_get(tfile); in tun_chr_poll() local
1428 if (!tun) in tun_chr_poll()
1443 if (tun_sock_writeable(tun, tfile) || in tun_chr_poll()
1445 tun_sock_writeable(tun, tfile))) in tun_chr_poll()
1448 if (tun->dev->reg_state != NETREG_REGISTERED) in tun_chr_poll()
1451 tun_put(tun); in tun_chr_poll()
1539 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile, in tun_rx_batched() argument
1544 u32 rx_batched = tun->rx_batched; in tun_rx_batched()
1579 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile, in tun_can_build_skb() argument
1582 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) in tun_can_build_skb()
1620 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, in tun_xdp_act() argument
1627 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); in tun_xdp_act()
1629 dev_core_stats_rx_dropped_inc(tun->dev); in tun_xdp_act()
1632 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data); in tun_xdp_act()
1635 err = tun_xdp_tx(tun->dev, xdp); in tun_xdp_act()
1637 dev_core_stats_rx_dropped_inc(tun->dev); in tun_xdp_act()
1640 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data); in tun_xdp_act()
1645 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); in tun_xdp_act()
1648 trace_xdp_exception(tun->dev, xdp_prog, act); in tun_xdp_act()
1651 dev_core_stats_rx_dropped_inc(tun->dev); in tun_xdp_act()
1658 static struct sk_buff *tun_build_skb(struct tun_struct *tun, in tun_build_skb() argument
1674 xdp_prog = rcu_dereference(tun->xdp_prog); in tun_build_skb()
1706 xdp_prog = rcu_dereference(tun->xdp_prog); in tun_build_skb()
1719 err = tun_xdp_act(tun, xdp_prog, &xdp, act); in tun_build_skb()
1748 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, in tun_get_user() argument
1755 size_t len = total_len, align = tun->align, linear; in tun_get_user()
1766 if (!(tun->flags & IFF_NO_PI)) { in tun_get_user()
1775 if (tun->flags & IFF_VNET_HDR) { in tun_get_user()
1776 int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); in tun_get_user()
1786 …tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso… in tun_get_user()
1787 …gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_off… in tun_get_user()
1789 if (tun16_to_cpu(tun, gso.hdr_len) > len) in tun_get_user()
1794 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) { in tun_get_user()
1797 (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN))) in tun_get_user()
1810 copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN; in tun_get_user()
1819 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) { in tun_get_user()
1824 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp); in tun_get_user()
1833 if (tun16_to_cpu(tun, gso.hdr_len) > good_linear) in tun_get_user()
1836 linear = tun16_to_cpu(tun, gso.hdr_len); in tun_get_user()
1871 if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) { in tun_get_user()
1872 atomic_long_inc(&tun->rx_frame_errors); in tun_get_user()
1877 switch (tun->flags & TUN_TYPE_MASK) { in tun_get_user()
1879 if (tun->flags & IFF_NO_PI) { in tun_get_user()
1897 skb->dev = tun->dev; in tun_get_user()
1905 skb->protocol = eth_type_trans(skb, tun->dev); in tun_get_user()
1927 xdp_prog = rcu_dereference(tun->xdp_prog); in tun_get_user()
1944 if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 && in tun_get_user()
1949 if (unlikely(!(tun->dev->flags & IFF_UP))) { in tun_get_user()
1961 headlen = eth_get_headlen(tun->dev, skb->data, in tun_get_user()
1967 dev_core_stats_rx_dropped_inc(tun->dev); in tun_get_user()
2007 tun_rx_batched(tun, tfile, skb, more); in tun_get_user()
2014 dev_sw_netstats_rx_add(tun->dev, len); in tun_get_user()
2018 tun_flow_update(tun, rxhash, tfile); in tun_get_user()
2024 dev_core_stats_rx_dropped_inc(tun->dev); in tun_get_user()
2043 struct tun_struct *tun = tun_get(tfile); in tun_chr_write_iter() local
2047 if (!tun) in tun_chr_write_iter()
2053 result = tun_get_user(tun, tfile, NULL, from, noblock, false); in tun_chr_write_iter()
2055 tun_put(tun); in tun_chr_write_iter()
2059 static ssize_t tun_put_user_xdp(struct tun_struct *tun, in tun_put_user_xdp() argument
2068 if (tun->flags & IFF_VNET_HDR) { in tun_put_user_xdp()
2071 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); in tun_put_user_xdp()
2083 dev_sw_netstats_tx_add(tun->dev, 1, ret); in tun_put_user_xdp()
2090 static ssize_t tun_put_user(struct tun_struct *tun, in tun_put_user() argument
2104 if (tun->flags & IFF_VNET_HDR) in tun_put_user()
2105 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); in tun_put_user()
2109 if (!(tun->flags & IFF_NO_PI)) { in tun_put_user()
2130 tun_is_little_endian(tun), true, in tun_put_user()
2135 netdev_err(tun->dev, "unexpected GSO type: 0x%x, gso_size %d, hdr_len %d\n", in tun_put_user()
2136 sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size), in tun_put_user()
2137 tun16_to_cpu(tun, gso.hdr_len)); in tun_put_user()
2141 min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true); in tun_put_user()
2176 dev_sw_netstats_tx_add(tun->dev, 1, skb->len + vlan_hlen); in tun_put_user()
2223 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, in tun_do_read() argument
2245 ret = tun_put_user_xdp(tun, tfile, xdpf, to); in tun_do_read()
2250 ret = tun_put_user(tun, tfile, skb, to); in tun_do_read()
2264 struct tun_struct *tun = tun_get(tfile); in tun_chr_read_iter() local
2268 if (!tun) in tun_chr_read_iter()
2274 ret = tun_do_read(tun, tfile, to, noblock, NULL); in tun_chr_read_iter()
2278 tun_put(tun); in tun_chr_read_iter()
2290 static int __tun_set_ebpf(struct tun_struct *tun, in __tun_set_ebpf() argument
2303 spin_lock_bh(&tun->lock); in __tun_set_ebpf()
2305 lockdep_is_held(&tun->lock)); in __tun_set_ebpf()
2307 spin_unlock_bh(&tun->lock); in __tun_set_ebpf()
2317 struct tun_struct *tun = netdev_priv(dev); in tun_free_netdev() local
2319 BUG_ON(!(list_empty(&tun->disabled))); in tun_free_netdev()
2321 tun_flow_uninit(tun); in tun_free_netdev()
2322 security_tun_dev_free_security(tun->security); in tun_free_netdev()
2323 __tun_set_ebpf(tun, &tun->steering_prog, NULL); in tun_free_netdev()
2324 __tun_set_ebpf(tun, &tun->filter_prog, NULL); in tun_free_netdev()
2329 struct tun_struct *tun = netdev_priv(dev); in tun_setup() local
2331 tun->owner = INVALID_UID; in tun_setup()
2332 tun->group = INVALID_GID; in tun_setup()
2333 tun_default_link_ksettings(dev, &tun->link_ksettings); in tun_setup()
2372 struct tun_struct *tun = netdev_priv(dev); in tun_fill_info() local
2374 if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK)) in tun_fill_info()
2376 if (uid_valid(tun->owner) && in tun_fill_info()
2378 from_kuid_munged(current_user_ns(), tun->owner))) in tun_fill_info()
2380 if (gid_valid(tun->group) && in tun_fill_info()
2382 from_kgid_munged(current_user_ns(), tun->group))) in tun_fill_info()
2384 if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI))) in tun_fill_info()
2386 if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR))) in tun_fill_info()
2388 if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST))) in tun_fill_info()
2391 !!(tun->flags & IFF_MULTI_QUEUE))) in tun_fill_info()
2393 if (tun->flags & IFF_MULTI_QUEUE) { in tun_fill_info()
2394 if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues)) in tun_fill_info()
2397 tun->numdisabled)) in tun_fill_info()
2442 static int tun_xdp_one(struct tun_struct *tun, in tun_xdp_one() argument
2462 xdp_prog = rcu_dereference(tun->xdp_prog); in tun_xdp_one()
2473 ret = tun_xdp_act(tun, xdp_prog, xdp, act); in tun_xdp_one()
2510 if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) { in tun_xdp_one()
2511 atomic_long_inc(&tun->rx_frame_errors); in tun_xdp_one()
2517 skb->protocol = eth_type_trans(skb, tun->dev); in tun_xdp_one()
2530 if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 && in tun_xdp_one()
2555 dev_sw_netstats_rx_add(tun->dev, datasize); in tun_xdp_one()
2558 tun_flow_update(tun, rxhash, tfile); in tun_xdp_one()
2568 struct tun_struct *tun = tun_get(tfile); in tun_sendmsg() local
2572 if (!tun) in tun_sendmsg()
2590 ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage); in tun_sendmsg()
2611 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter, in tun_sendmsg()
2615 tun_put(tun); in tun_sendmsg()
2623 struct tun_struct *tun = tun_get(tfile); in tun_recvmsg() local
2627 if (!tun) { in tun_recvmsg()
2641 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr); in tun_recvmsg()
2647 tun_put(tun); in tun_recvmsg()
2651 tun_put(tun); in tun_recvmsg()
2674 struct tun_struct *tun; in tun_peek_len() local
2677 tun = tun_get(tfile); in tun_peek_len()
2678 if (!tun) in tun_peek_len()
2682 tun_put(tun); in tun_peek_len()
2700 static int tun_flags(struct tun_struct *tun) in tun_flags() argument
2702 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP); in tun_flags()
2708 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); in tun_flags_show() local
2709 return sysfs_emit(buf, "0x%x\n", tun_flags(tun)); in tun_flags_show()
2715 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); in owner_show() local
2716 return uid_valid(tun->owner)? in owner_show()
2718 from_kuid_munged(current_user_ns(), tun->owner)) : in owner_show()
2725 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); in group_show() local
2726 return gid_valid(tun->group) ? in group_show()
2728 from_kgid_munged(current_user_ns(), tun->group)) : in group_show()
2749 struct tun_struct *tun; in tun_set_iff() local
2771 tun = netdev_priv(dev); in tun_set_iff()
2773 tun = netdev_priv(dev); in tun_set_iff()
2778 !!(tun->flags & IFF_MULTI_QUEUE)) in tun_set_iff()
2781 if (tun_not_capable(tun)) in tun_set_iff()
2783 err = security_tun_dev_open(tun->security); in tun_set_iff()
2787 err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER, in tun_set_iff()
2793 if (tun->flags & IFF_MULTI_QUEUE && in tun_set_iff()
2794 (tun->numqueues + tun->numdisabled > 1)) { in tun_set_iff()
2802 tun->flags = (tun->flags & ~TUN_FEATURES) | in tun_set_iff()
2845 tun = netdev_priv(dev); in tun_set_iff()
2846 tun->dev = dev; in tun_set_iff()
2847 tun->flags = flags; in tun_set_iff()
2848 tun->txflt.count = 0; in tun_set_iff()
2849 tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr); in tun_set_iff()
2851 tun->align = NET_SKB_PAD; in tun_set_iff()
2852 tun->filter_attached = false; in tun_set_iff()
2853 tun->sndbuf = tfile->socket.sk->sk_sndbuf; in tun_set_iff()
2854 tun->rx_batched = 0; in tun_set_iff()
2855 RCU_INIT_POINTER(tun->steering_prog, NULL); in tun_set_iff()
2857 tun->ifr = ifr; in tun_set_iff()
2858 tun->file = file; in tun_set_iff()
2862 err = register_netdevice(tun->dev); in tun_set_iff()
2870 rcu_assign_pointer(tfile->tun, tun); in tun_set_iff()
2874 netif_carrier_off(tun->dev); in tun_set_iff()
2876 netif_carrier_on(tun->dev); in tun_set_iff()
2881 if (netif_running(tun->dev)) in tun_set_iff()
2882 netif_tx_wake_all_queues(tun->dev); in tun_set_iff()
2884 strcpy(ifr->ifr_name, tun->dev->name); in tun_set_iff()
2888 static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr) in tun_get_iff() argument
2890 strcpy(ifr->ifr_name, tun->dev->name); in tun_get_iff()
2892 ifr->ifr_flags = tun_flags(tun); in tun_get_iff()
2898 static int set_offload(struct tun_struct *tun, unsigned long arg) in set_offload() argument
2932 tun->set_features = features; in set_offload()
2933 tun->dev->wanted_features &= ~TUN_USER_FEATURES; in set_offload()
2934 tun->dev->wanted_features |= features; in set_offload()
2935 netdev_update_features(tun->dev); in set_offload()
2940 static void tun_detach_filter(struct tun_struct *tun, int n) in tun_detach_filter() argument
2946 tfile = rtnl_dereference(tun->tfiles[i]); in tun_detach_filter()
2952 tun->filter_attached = false; in tun_detach_filter()
2955 static int tun_attach_filter(struct tun_struct *tun) in tun_attach_filter() argument
2960 for (i = 0; i < tun->numqueues; i++) { in tun_attach_filter()
2961 tfile = rtnl_dereference(tun->tfiles[i]); in tun_attach_filter()
2963 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); in tun_attach_filter()
2966 tun_detach_filter(tun, i); in tun_attach_filter()
2971 tun->filter_attached = true; in tun_attach_filter()
2975 static void tun_set_sndbuf(struct tun_struct *tun) in tun_set_sndbuf() argument
2980 for (i = 0; i < tun->numqueues; i++) { in tun_set_sndbuf()
2981 tfile = rtnl_dereference(tun->tfiles[i]); in tun_set_sndbuf()
2982 tfile->socket.sk->sk_sndbuf = tun->sndbuf; in tun_set_sndbuf()
2989 struct tun_struct *tun; in tun_set_queue() local
2995 tun = tfile->detached; in tun_set_queue()
2996 if (!tun) { in tun_set_queue()
3000 ret = security_tun_dev_attach_queue(tun->security); in tun_set_queue()
3003 ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI, in tun_set_queue()
3004 tun->flags & IFF_NAPI_FRAGS, true); in tun_set_queue()
3006 tun = rtnl_dereference(tfile->tun); in tun_set_queue()
3007 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached) in tun_set_queue()
3015 netdev_state_change(tun->dev); in tun_set_queue()
3022 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p, in tun_set_ebpf() argument
3039 return __tun_set_ebpf(tun, prog_p, prog); in tun_set_ebpf()
3086 struct tun_struct *tun; in __tun_chr_ioctl() local
3123 tun = tun_get(tfile); in __tun_chr_ioctl()
3126 if (tun) in __tun_chr_ioctl()
3142 if (tun) in __tun_chr_ioctl()
3157 if (!tun) in __tun_chr_ioctl()
3160 netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd); in __tun_chr_ioctl()
3162 net = dev_net(tun->dev); in __tun_chr_ioctl()
3166 tun_get_iff(tun, &ifr); in __tun_chr_ioctl()
3181 netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n", in __tun_chr_ioctl()
3189 if (arg && !(tun->flags & IFF_PERSIST)) { in __tun_chr_ioctl()
3190 tun->flags |= IFF_PERSIST; in __tun_chr_ioctl()
3194 if (!arg && (tun->flags & IFF_PERSIST)) { in __tun_chr_ioctl()
3195 tun->flags &= ~IFF_PERSIST; in __tun_chr_ioctl()
3200 netif_info(tun, drv, tun->dev, "persist %s\n", in __tun_chr_ioctl()
3211 tun->owner = owner; in __tun_chr_ioctl()
3213 netif_info(tun, drv, tun->dev, "owner set to %u\n", in __tun_chr_ioctl()
3214 from_kuid(&init_user_ns, tun->owner)); in __tun_chr_ioctl()
3224 tun->group = group; in __tun_chr_ioctl()
3226 netif_info(tun, drv, tun->dev, "group set to %u\n", in __tun_chr_ioctl()
3227 from_kgid(&init_user_ns, tun->group)); in __tun_chr_ioctl()
3232 if (tun->dev->flags & IFF_UP) { in __tun_chr_ioctl()
3233 netif_info(tun, drv, tun->dev, in __tun_chr_ioctl()
3238 tun->dev); in __tun_chr_ioctl()
3241 netif_info(tun, drv, tun->dev, in __tun_chr_ioctl()
3245 tun->dev->type = (int) arg; in __tun_chr_ioctl()
3246 tun->dev->addr_len = tun_get_addr_len(tun->dev->type); in __tun_chr_ioctl()
3247 netif_info(tun, drv, tun->dev, "linktype set to %d\n", in __tun_chr_ioctl()
3248 tun->dev->type); in __tun_chr_ioctl()
3250 tun->dev); in __tun_chr_ioctl()
3255 tun->msg_enable = (u32)arg; in __tun_chr_ioctl()
3259 ret = set_offload(tun, arg); in __tun_chr_ioctl()
3265 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) in __tun_chr_ioctl()
3267 ret = update_filter(&tun->txflt, (void __user *)arg); in __tun_chr_ioctl()
3272 dev_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name); in __tun_chr_ioctl()
3279 ret = dev_set_mac_address_user(tun->dev, &ifr.ifr_hwaddr, NULL); in __tun_chr_ioctl()
3298 tun->sndbuf = sndbuf; in __tun_chr_ioctl()
3299 tun_set_sndbuf(tun); in __tun_chr_ioctl()
3303 vnet_hdr_sz = tun->vnet_hdr_sz; in __tun_chr_ioctl()
3318 tun->vnet_hdr_sz = vnet_hdr_sz; in __tun_chr_ioctl()
3322 le = !!(tun->flags & TUN_VNET_LE); in __tun_chr_ioctl()
3333 tun->flags |= TUN_VNET_LE; in __tun_chr_ioctl()
3335 tun->flags &= ~TUN_VNET_LE; in __tun_chr_ioctl()
3339 ret = tun_get_vnet_be(tun, argp); in __tun_chr_ioctl()
3343 ret = tun_set_vnet_be(tun, argp); in __tun_chr_ioctl()
3349 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) in __tun_chr_ioctl()
3352 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog))) in __tun_chr_ioctl()
3355 ret = tun_attach_filter(tun); in __tun_chr_ioctl()
3361 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) in __tun_chr_ioctl()
3364 tun_detach_filter(tun, tun->numqueues); in __tun_chr_ioctl()
3369 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) in __tun_chr_ioctl()
3372 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog))) in __tun_chr_ioctl()
3378 ret = tun_set_ebpf(tun, &tun->steering_prog, argp); in __tun_chr_ioctl()
3382 ret = tun_set_ebpf(tun, &tun->filter_prog, argp); in __tun_chr_ioctl()
3390 ret = tun_net_change_carrier(tun->dev, (bool)carrier); in __tun_chr_ioctl()
3406 netdev_state_change(tun->dev); in __tun_chr_ioctl()
3410 if (tun) in __tun_chr_ioctl()
3411 tun_put(tun); in __tun_chr_ioctl()
3489 RCU_INIT_POINTER(tfile->tun, NULL); in tun_chr_open()
3526 struct tun_struct *tun; in tun_chr_show_fdinfo() local
3532 tun = tun_get(tfile); in tun_chr_show_fdinfo()
3533 if (tun) in tun_chr_show_fdinfo()
3534 tun_get_iff(tun, &ifr); in tun_chr_show_fdinfo()
3537 if (tun) in tun_chr_show_fdinfo()
3538 tun_put(tun); in tun_chr_show_fdinfo()
3585 struct tun_struct *tun = netdev_priv(dev); in tun_get_link_ksettings() local
3587 memcpy(cmd, &tun->link_ksettings, sizeof(*cmd)); in tun_get_link_ksettings()
3594 struct tun_struct *tun = netdev_priv(dev); in tun_set_link_ksettings() local
3596 memcpy(&tun->link_ksettings, cmd, sizeof(*cmd)); in tun_set_link_ksettings()
3602 struct tun_struct *tun = netdev_priv(dev); in tun_get_drvinfo() local
3607 switch (tun->flags & TUN_TYPE_MASK) { in tun_get_drvinfo()
3619 struct tun_struct *tun = netdev_priv(dev); in tun_get_msglevel() local
3621 return tun->msg_enable; in tun_get_msglevel()
3626 struct tun_struct *tun = netdev_priv(dev); in tun_set_msglevel() local
3628 tun->msg_enable = value; in tun_set_msglevel()
3636 struct tun_struct *tun = netdev_priv(dev); in tun_get_coalesce() local
3638 ec->rx_max_coalesced_frames = tun->rx_batched; in tun_get_coalesce()
3648 struct tun_struct *tun = netdev_priv(dev); in tun_set_coalesce() local
3651 tun->rx_batched = NAPI_POLL_WEIGHT; in tun_set_coalesce()
3653 tun->rx_batched = ec->rx_max_coalesced_frames; in tun_set_coalesce()
3661 struct tun_struct *tun = netdev_priv(dev); in tun_get_channels() local
3663 channels->combined_count = tun->numqueues; in tun_get_channels()
3664 channels->max_combined = tun->flags & IFF_MULTI_QUEUE ? MAX_TAP_QUEUES : 1; in tun_get_channels()
3681 static int tun_queue_resize(struct tun_struct *tun) in tun_queue_resize() argument
3683 struct net_device *dev = tun->dev; in tun_queue_resize()
3686 int n = tun->numqueues + tun->numdisabled; in tun_queue_resize()
3693 for (i = 0; i < tun->numqueues; i++) { in tun_queue_resize()
3694 tfile = rtnl_dereference(tun->tfiles[i]); in tun_queue_resize()
3697 list_for_each_entry(tfile, &tun->disabled, next) in tun_queue_resize()
3712 struct tun_struct *tun = netdev_priv(dev); in tun_device_event() local
3720 if (tun_queue_resize(tun)) in tun_device_event()
3724 for (i = 0; i < tun->numqueues; i++) { in tun_device_event()
3727 tfile = rtnl_dereference(tun->tfiles[i]); in tun_device_event()