Lines Matching +full:data +full:- +full:out
1 // SPDX-License-Identifier: GPL-2.0-only
20 #include <linux/error-injection.h>
41 if (t->mode == NO_PREEMPT) in bpf_test_timer_enter()
46 t->time_start = ktime_get_ns(); in bpf_test_timer_enter()
52 t->time_start = 0; in bpf_test_timer_leave()
54 if (t->mode == NO_PREEMPT) in bpf_test_timer_leave()
65 t->i += iterations; in bpf_test_timer_continue()
66 if (t->i >= repeat) { in bpf_test_timer_continue()
68 t->time_spent += ktime_get_ns() - t->time_start; in bpf_test_timer_continue()
69 do_div(t->time_spent, t->i); in bpf_test_timer_continue()
70 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; in bpf_test_timer_continue()
77 *err = -EINTR; in bpf_test_timer_continue()
83 t->time_spent += ktime_get_ns() - t->time_start; in bpf_test_timer_continue()
93 t->i = 0; in bpf_test_timer_continue()
107 DECLARE_FLEX_ARRAY(u8, data);
127 #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
139 void *data; in xdp_test_run_init_page() local
141 orig_ctx = xdp->orig_ctx; in xdp_test_run_init_page()
142 frm_len = orig_ctx->data_end - orig_ctx->data_meta; in xdp_test_run_init_page()
143 meta_len = orig_ctx->data - orig_ctx->data_meta; in xdp_test_run_init_page()
144 headroom -= meta_len; in xdp_test_run_init_page()
146 new_ctx = &head->ctx; in xdp_test_run_init_page()
147 frm = head->frame; in xdp_test_run_init_page()
148 data = head->data; in xdp_test_run_init_page()
149 memcpy(data + headroom, orig_ctx->data_meta, frm_len); in xdp_test_run_init_page()
151 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); in xdp_test_run_init_page()
152 xdp_prepare_buff(new_ctx, data, headroom, frm_len, true); in xdp_test_run_init_page()
153 new_ctx->data = new_ctx->data_meta + meta_len; in xdp_test_run_init_page()
156 frm->mem = new_ctx->rxq->mem; in xdp_test_run_init_page()
158 memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx)); in xdp_test_run_init_page()
164 int err = -ENOMEM; in xdp_test_run_setup()
168 .pool_size = xdp->batch_size, in xdp_test_run_setup()
174 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
175 if (!xdp->frames) in xdp_test_run_setup()
176 return -ENOMEM; in xdp_test_run_setup()
178 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
179 if (!xdp->skbs) in xdp_test_run_setup()
188 /* will copy 'mem.id' into pp->xdp_mem_id */ in xdp_test_run_setup()
189 err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp); in xdp_test_run_setup()
193 xdp->pp = pp; in xdp_test_run_setup()
198 xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0); in xdp_test_run_setup()
199 xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL; in xdp_test_run_setup()
200 xdp->rxq.mem.id = pp->xdp_mem_id; in xdp_test_run_setup()
201 xdp->dev = orig_ctx->rxq->dev; in xdp_test_run_setup()
202 xdp->orig_ctx = orig_ctx; in xdp_test_run_setup()
209 kvfree(xdp->skbs); in xdp_test_run_setup()
211 kvfree(xdp->frames); in xdp_test_run_setup()
217 xdp_unreg_mem_model(&xdp->mem); in xdp_test_run_teardown()
218 page_pool_destroy(xdp->pp); in xdp_test_run_teardown()
219 kfree(xdp->frames); in xdp_test_run_teardown()
220 kfree(xdp->skbs); in xdp_test_run_teardown()
225 /* xdp_scrub_frame() zeroes the data pointer, flags is the last field, in frame_was_changed()
229 return head->frame->data != head->orig_ctx.data || in frame_was_changed()
230 head->frame->flags != head->orig_ctx.flags; in frame_was_changed()
235 return head->orig_ctx.data != head->ctx.data || in ctx_was_changed()
236 head->orig_ctx.data_meta != head->ctx.data_meta || in ctx_was_changed()
237 head->orig_ctx.data_end != head->ctx.data_end; in ctx_was_changed()
245 head->ctx.data = head->orig_ctx.data; in reset_ctx()
246 head->ctx.data_meta = head->orig_ctx.data_meta; in reset_ctx()
247 head->ctx.data_end = head->orig_ctx.data_end; in reset_ctx()
248 xdp_update_frame_from_buff(&head->ctx, head->frame); in reset_ctx()
249 head->frame->mem = head->orig_ctx.rxq->mem; in reset_ctx()
265 return -ENOMEM; in xdp_recv_frames()
278 list_add_tail(&skb->list, &list); in xdp_recv_frames()
290 struct xdp_frame **frames = xdp->frames; in xdp_test_run_batch()
298 batch_sz = min_t(u32, repeat, xdp->batch_size); in xdp_test_run_batch()
306 page = page_pool_dev_alloc_pages(xdp->pp); in xdp_test_run_batch()
308 err = -ENOMEM; in xdp_test_run_batch()
309 goto out; in xdp_test_run_batch()
314 ctx = &head->ctx; in xdp_test_run_batch()
315 frm = head->frame; in xdp_test_run_batch()
316 xdp->frame_cnt++; in xdp_test_run_batch()
335 ri->tgt_index = xdp->dev->ifindex; in xdp_test_run_batch()
336 ri->map_id = INT_MAX; in xdp_test_run_batch()
337 ri->map_type = BPF_MAP_TYPE_UNSPEC; in xdp_test_run_batch()
341 ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog); in xdp_test_run_batch()
357 out: in xdp_test_run_batch()
361 ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev); in xdp_test_run_batch()
390 ret = xdp_test_run_batch(&xdp, prog, repeat - t.i); in bpf_test_run_xdp_live()
417 return -ENOMEM; in bpf_test_run()
449 union bpf_attr __user *uattr, const void *data, in bpf_test_finish() argument
453 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); in bpf_test_finish()
454 int err = -EFAULT; in bpf_test_finish()
460 if (kattr->test.data_size_out && in bpf_test_finish()
461 copy_size > kattr->test.data_size_out) { in bpf_test_finish()
462 copy_size = kattr->test.data_size_out; in bpf_test_finish()
463 err = -ENOSPC; in bpf_test_finish()
467 int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size; in bpf_test_finish()
470 err = -ENOSPC; in bpf_test_finish()
471 goto out; in bpf_test_finish()
474 if (copy_to_user(data_out, data, len)) in bpf_test_finish()
475 goto out; in bpf_test_finish()
481 for (i = 0; i < sinfo->nr_frags; i++) { in bpf_test_finish()
482 skb_frag_t *frag = &sinfo->frags[i]; in bpf_test_finish()
485 err = -ENOSPC; in bpf_test_finish()
489 data_len = min_t(u32, copy_size - offset, in bpf_test_finish()
495 goto out; in bpf_test_finish()
502 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) in bpf_test_finish()
503 goto out; in bpf_test_finish()
504 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_test_finish()
505 goto out; in bpf_test_finish()
506 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) in bpf_test_finish()
507 goto out; in bpf_test_finish()
508 if (err != -ENOSPC) in bpf_test_finish()
510 out: in bpf_test_finish()
564 return (long)arg->a; in bpf_fentry_test8()
620 refcount_dec(&p->cnt); in bpf_kfunc_call_test_release()
660 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); in BTF_ID_FLAGS()
661 void *data; in BTF_ID_FLAGS() local
663 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) in BTF_ID_FLAGS()
664 return ERR_PTR(-EINVAL); in BTF_ID_FLAGS()
667 return ERR_PTR(-EMSGSIZE); in BTF_ID_FLAGS()
670 data = kzalloc(size + headroom + tailroom, GFP_USER); in BTF_ID_FLAGS()
671 if (!data) in BTF_ID_FLAGS()
672 return ERR_PTR(-ENOMEM); in BTF_ID_FLAGS()
674 if (copy_from_user(data + headroom, data_in, user_size)) { in BTF_ID_FLAGS()
675 kfree(data); in BTF_ID_FLAGS()
676 return ERR_PTR(-EFAULT); in BTF_ID_FLAGS()
679 return data; in BTF_ID_FLAGS()
688 int b = 2, err = -EFAULT; in bpf_prog_test_run_tracing()
691 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_tracing()
692 return -EINVAL; in bpf_prog_test_run_tracing()
694 switch (prog->expected_attach_type) { in bpf_prog_test_run_tracing()
706 goto out; in bpf_prog_test_run_tracing()
718 goto out; in bpf_prog_test_run_tracing()
722 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_prog_test_run_tracing()
723 goto out; in bpf_prog_test_run_tracing()
726 out: in bpf_prog_test_run_tracing()
738 __bpf_prog_test_run_raw_tp(void *data) in __bpf_prog_test_run_raw_tp() argument
740 struct bpf_raw_tp_test_run_info *info = data; in __bpf_prog_test_run_raw_tp()
747 info->retval = bpf_prog_run(info->prog, info->ctx); in __bpf_prog_test_run_raw_tp()
757 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_prog_test_run_raw_tp()
758 __u32 ctx_size_in = kattr->test.ctx_size_in; in bpf_prog_test_run_raw_tp()
760 int cpu = kattr->test.cpu, err = 0; in bpf_prog_test_run_raw_tp()
763 /* doesn't support data_in/out, ctx_out, duration, or repeat */ in bpf_prog_test_run_raw_tp()
764 if (kattr->test.data_in || kattr->test.data_out || in bpf_prog_test_run_raw_tp()
765 kattr->test.ctx_out || kattr->test.duration || in bpf_prog_test_run_raw_tp()
766 kattr->test.repeat || kattr->test.batch_size) in bpf_prog_test_run_raw_tp()
767 return -EINVAL; in bpf_prog_test_run_raw_tp()
769 if (ctx_size_in < prog->aux->max_ctx_offset || in bpf_prog_test_run_raw_tp()
771 return -EINVAL; in bpf_prog_test_run_raw_tp()
773 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) in bpf_prog_test_run_raw_tp()
774 return -EINVAL; in bpf_prog_test_run_raw_tp()
787 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || in bpf_prog_test_run_raw_tp()
793 * space, let's do an extra quick check to filter out in bpf_prog_test_run_raw_tp()
796 err = -ENXIO; in bpf_prog_test_run_raw_tp()
804 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) in bpf_prog_test_run_raw_tp()
805 err = -EFAULT; in bpf_prog_test_run_raw_tp()
813 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_ctx_init()
814 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_init()
815 u32 size = kattr->test.ctx_size_in; in bpf_ctx_init()
816 void *data; in bpf_ctx_init() local
822 data = kzalloc(max_size, GFP_USER); in bpf_ctx_init()
823 if (!data) in bpf_ctx_init()
824 return ERR_PTR(-ENOMEM); in bpf_ctx_init()
829 kfree(data); in bpf_ctx_init()
834 if (copy_from_user(data, data_in, size)) { in bpf_ctx_init()
835 kfree(data); in bpf_ctx_init()
836 return ERR_PTR(-EFAULT); in bpf_ctx_init()
839 return data; in bpf_ctx_init()
843 union bpf_attr __user *uattr, const void *data, in bpf_ctx_finish() argument
846 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_finish()
847 int err = -EFAULT; in bpf_ctx_finish()
850 if (!data || !data_out) in bpf_ctx_finish()
853 if (copy_size > kattr->test.ctx_size_out) { in bpf_ctx_finish()
854 copy_size = kattr->test.ctx_size_out; in bpf_ctx_finish()
855 err = -ENOSPC; in bpf_ctx_finish()
858 if (copy_to_user(data_out, data, copy_size)) in bpf_ctx_finish()
859 goto out; in bpf_ctx_finish()
860 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) in bpf_ctx_finish()
861 goto out; in bpf_ctx_finish()
862 if (err != -ENOSPC) in bpf_ctx_finish()
864 out: in bpf_ctx_finish()
869 * range_is_zero - test whether buffer is initialized
874 * This function returns true if the there is a non-zero byte
879 return !memchr_inv((u8 *)buf + from, 0, to - from); in range_is_zero()
884 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert___skb_to_skb()
891 return -EINVAL; in convert___skb_to_skb()
897 return -EINVAL; in convert___skb_to_skb()
905 return -EINVAL; in convert___skb_to_skb()
911 return -EINVAL; in convert___skb_to_skb()
919 return -EINVAL; in convert___skb_to_skb()
925 return -EINVAL; in convert___skb_to_skb()
931 return -EINVAL; in convert___skb_to_skb()
933 skb->mark = __skb->mark; in convert___skb_to_skb()
934 skb->priority = __skb->priority; in convert___skb_to_skb()
935 skb->skb_iif = __skb->ingress_ifindex; in convert___skb_to_skb()
936 skb->tstamp = __skb->tstamp; in convert___skb_to_skb()
937 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); in convert___skb_to_skb()
939 if (__skb->wire_len == 0) { in convert___skb_to_skb()
940 cb->pkt_len = skb->len; in convert___skb_to_skb()
942 if (__skb->wire_len < skb->len || in convert___skb_to_skb()
943 __skb->wire_len > GSO_LEGACY_MAX_SIZE) in convert___skb_to_skb()
944 return -EINVAL; in convert___skb_to_skb()
945 cb->pkt_len = __skb->wire_len; in convert___skb_to_skb()
948 if (__skb->gso_segs > GSO_MAX_SEGS) in convert___skb_to_skb()
949 return -EINVAL; in convert___skb_to_skb()
950 skb_shinfo(skb)->gso_segs = __skb->gso_segs; in convert___skb_to_skb()
951 skb_shinfo(skb)->gso_size = __skb->gso_size; in convert___skb_to_skb()
952 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; in convert___skb_to_skb()
959 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert_skb_to___skb()
964 __skb->mark = skb->mark; in convert_skb_to___skb()
965 __skb->priority = skb->priority; in convert_skb_to___skb()
966 __skb->ingress_ifindex = skb->skb_iif; in convert_skb_to___skb()
967 __skb->ifindex = skb->dev->ifindex; in convert_skb_to___skb()
968 __skb->tstamp = skb->tstamp; in convert_skb_to___skb()
969 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); in convert_skb_to___skb()
970 __skb->wire_len = cb->pkt_len; in convert_skb_to___skb()
971 __skb->gso_segs = skb_shinfo(skb)->gso_segs; in convert_skb_to___skb()
972 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; in convert_skb_to___skb()
985 struct net *net = current->nsproxy->net_ns; in bpf_prog_test_run_skb()
986 struct net_device *dev = net->loopback_dev; in bpf_prog_test_run_skb()
987 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_skb()
988 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_skb()
994 void *data; in bpf_prog_test_run_skb() local
997 if ((kattr->test.flags & ~BPF_F_TEST_SKB_CHECKSUM_COMPLETE) || in bpf_prog_test_run_skb()
998 kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_skb()
999 return -EINVAL; in bpf_prog_test_run_skb()
1001 data = bpf_test_init(kattr, kattr->test.data_size_in, in bpf_prog_test_run_skb()
1004 if (IS_ERR(data)) in bpf_prog_test_run_skb()
1005 return PTR_ERR(data); in bpf_prog_test_run_skb()
1009 kfree(data); in bpf_prog_test_run_skb()
1013 switch (prog->type) { in bpf_prog_test_run_skb()
1029 kfree(data); in bpf_prog_test_run_skb()
1031 return -ENOMEM; in bpf_prog_test_run_skb()
1035 skb = slab_build_skb(data); in bpf_prog_test_run_skb()
1037 kfree(data); in bpf_prog_test_run_skb()
1040 return -ENOMEM; in bpf_prog_test_run_skb()
1042 skb->sk = sk; in bpf_prog_test_run_skb()
1047 if (ctx && ctx->ifindex > 1) { in bpf_prog_test_run_skb()
1048 dev = dev_get_by_index(net, ctx->ifindex); in bpf_prog_test_run_skb()
1050 ret = -ENODEV; in bpf_prog_test_run_skb()
1051 goto out; in bpf_prog_test_run_skb()
1054 skb->protocol = eth_type_trans(skb, dev); in bpf_prog_test_run_skb()
1057 switch (skb->protocol) { in bpf_prog_test_run_skb()
1059 sk->sk_family = AF_INET; in bpf_prog_test_run_skb()
1061 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; in bpf_prog_test_run_skb()
1062 sk->sk_daddr = ip_hdr(skb)->daddr; in bpf_prog_test_run_skb()
1067 sk->sk_family = AF_INET6; in bpf_prog_test_run_skb()
1069 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; in bpf_prog_test_run_skb()
1070 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; in bpf_prog_test_run_skb()
1085 goto out; in bpf_prog_test_run_skb()
1087 if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) { in bpf_prog_test_run_skb()
1089 int len = skb->len - off; in bpf_prog_test_run_skb()
1091 skb->csum = skb_checksum(skb, off, len, 0); in bpf_prog_test_run_skb()
1092 skb->ip_summed = CHECKSUM_COMPLETE; in bpf_prog_test_run_skb()
1097 goto out; in bpf_prog_test_run_skb()
1100 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); in bpf_prog_test_run_skb()
1103 ret = -ENOMEM; in bpf_prog_test_run_skb()
1104 goto out; in bpf_prog_test_run_skb()
1110 if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) { in bpf_prog_test_run_skb()
1112 int len = skb->len - off; in bpf_prog_test_run_skb()
1117 if (csum_fold(skb->csum) != csum_fold(csum)) { in bpf_prog_test_run_skb()
1118 ret = -EBADMSG; in bpf_prog_test_run_skb()
1119 goto out; in bpf_prog_test_run_skb()
1125 size = skb->len; in bpf_prog_test_run_skb()
1126 /* bpf program can never convert linear skb to non-linear */ in bpf_prog_test_run_skb()
1129 ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval, in bpf_prog_test_run_skb()
1134 out: in bpf_prog_test_run_skb()
1135 if (dev && dev != net->loopback_dev) in bpf_prog_test_run_skb()
1152 if (xdp_md->egress_ifindex != 0) in xdp_convert_md_to_buff()
1153 return -EINVAL; in xdp_convert_md_to_buff()
1155 ingress_ifindex = xdp_md->ingress_ifindex; in xdp_convert_md_to_buff()
1156 rx_queue_index = xdp_md->rx_queue_index; in xdp_convert_md_to_buff()
1159 return -EINVAL; in xdp_convert_md_to_buff()
1162 device = dev_get_by_index(current->nsproxy->net_ns, in xdp_convert_md_to_buff()
1165 return -ENODEV; in xdp_convert_md_to_buff()
1167 if (rx_queue_index >= device->real_num_rx_queues) in xdp_convert_md_to_buff()
1172 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) in xdp_convert_md_to_buff()
1175 xdp->rxq = &rxqueue->xdp_rxq; in xdp_convert_md_to_buff()
1176 /* The device is now tracked in the xdp->rxq for later in xdp_convert_md_to_buff()
1181 xdp->data = xdp->data_meta + xdp_md->data; in xdp_convert_md_to_buff()
1186 return -EINVAL; in xdp_convert_md_to_buff()
1194 xdp_md->data = xdp->data - xdp->data_meta; in xdp_convert_buff_to_md()
1195 xdp_md->data_end = xdp->data_end - xdp->data_meta; in xdp_convert_buff_to_md()
1197 if (xdp_md->ingress_ifindex) in xdp_convert_buff_to_md()
1198 dev_put(xdp->rxq->dev); in xdp_convert_buff_to_md()
1204 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); in bpf_prog_test_run_xdp()
1206 u32 batch_size = kattr->test.batch_size; in bpf_prog_test_run_xdp()
1208 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_xdp()
1210 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_xdp()
1214 int i, ret = -EINVAL; in bpf_prog_test_run_xdp()
1216 void *data; in bpf_prog_test_run_xdp() local
1218 if (prog->expected_attach_type == BPF_XDP_DEVMAP || in bpf_prog_test_run_xdp()
1219 prog->expected_attach_type == BPF_XDP_CPUMAP) in bpf_prog_test_run_xdp()
1220 return -EINVAL; in bpf_prog_test_run_xdp()
1222 if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES) in bpf_prog_test_run_xdp()
1223 return -EINVAL; in bpf_prog_test_run_xdp()
1225 if (bpf_prog_is_dev_bound(prog->aux)) in bpf_prog_test_run_xdp()
1226 return -EINVAL; in bpf_prog_test_run_xdp()
1232 return -E2BIG; in bpf_prog_test_run_xdp()
1236 return -EINVAL; in bpf_prog_test_run_xdp()
1244 /* There can't be user provided data before the meta data */ in bpf_prog_test_run_xdp()
1245 if (ctx->data_meta || ctx->data_end != size || in bpf_prog_test_run_xdp()
1246 ctx->data > ctx->data_end || in bpf_prog_test_run_xdp()
1247 unlikely(xdp_metalen_invalid(ctx->data)) || in bpf_prog_test_run_xdp()
1248 (do_live && (kattr->test.data_out || kattr->test.ctx_out))) in bpf_prog_test_run_xdp()
1250 /* Meta data is allocated from the headroom */ in bpf_prog_test_run_xdp()
1251 headroom -= ctx->data; in bpf_prog_test_run_xdp()
1254 max_data_sz = 4096 - headroom - tailroom; in bpf_prog_test_run_xdp()
1256 /* disallow live data mode for jumbo frames */ in bpf_prog_test_run_xdp()
1262 data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); in bpf_prog_test_run_xdp()
1263 if (IS_ERR(data)) { in bpf_prog_test_run_xdp()
1264 ret = PTR_ERR(data); in bpf_prog_test_run_xdp()
1268 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); in bpf_prog_test_run_xdp()
1269 rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom; in bpf_prog_test_run_xdp()
1270 xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq); in bpf_prog_test_run_xdp()
1271 xdp_prepare_buff(&xdp, data, headroom, size, true); in bpf_prog_test_run_xdp()
1278 if (unlikely(kattr->test.data_size_in > size)) { in bpf_prog_test_run_xdp()
1279 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); in bpf_prog_test_run_xdp()
1281 while (size < kattr->test.data_size_in) { in bpf_prog_test_run_xdp()
1286 if (sinfo->nr_frags == MAX_SKB_FRAGS) { in bpf_prog_test_run_xdp()
1287 ret = -ENOMEM; in bpf_prog_test_run_xdp()
1288 goto out; in bpf_prog_test_run_xdp()
1293 ret = -ENOMEM; in bpf_prog_test_run_xdp()
1294 goto out; in bpf_prog_test_run_xdp()
1297 frag = &sinfo->frags[sinfo->nr_frags++]; in bpf_prog_test_run_xdp()
1299 data_len = min_t(u32, kattr->test.data_size_in - size, in bpf_prog_test_run_xdp()
1305 ret = -EFAULT; in bpf_prog_test_run_xdp()
1306 goto out; in bpf_prog_test_run_xdp()
1308 sinfo->xdp_frags_size += data_len; in bpf_prog_test_run_xdp()
1327 goto out; in bpf_prog_test_run_xdp()
1329 size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size; in bpf_prog_test_run_xdp()
1336 out: in bpf_prog_test_run_xdp()
1340 for (i = 0; i < sinfo->nr_frags; i++) in bpf_prog_test_run_xdp()
1341 __free_page(skb_frag_page(&sinfo->frags[i])); in bpf_prog_test_run_xdp()
1342 kfree(data); in bpf_prog_test_run_xdp()
1352 return -EINVAL; in verify_user_bpf_flow_keys()
1358 return -EINVAL; in verify_user_bpf_flow_keys()
1368 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_flow_dissector()
1370 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_flow_dissector()
1376 void *data; in bpf_prog_test_run_flow_dissector() local
1379 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_flow_dissector()
1380 return -EINVAL; in bpf_prog_test_run_flow_dissector()
1383 return -EINVAL; in bpf_prog_test_run_flow_dissector()
1385 data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0); in bpf_prog_test_run_flow_dissector()
1386 if (IS_ERR(data)) in bpf_prog_test_run_flow_dissector()
1387 return PTR_ERR(data); in bpf_prog_test_run_flow_dissector()
1389 eth = (struct ethhdr *)data; in bpf_prog_test_run_flow_dissector()
1396 kfree(data); in bpf_prog_test_run_flow_dissector()
1402 goto out; in bpf_prog_test_run_flow_dissector()
1403 flags = user_ctx->flags; in bpf_prog_test_run_flow_dissector()
1407 ctx.data = data; in bpf_prog_test_run_flow_dissector()
1408 ctx.data_end = (__u8 *)data + size; in bpf_prog_test_run_flow_dissector()
1412 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, in bpf_prog_test_run_flow_dissector()
1418 goto out; in bpf_prog_test_run_flow_dissector()
1426 out: in bpf_prog_test_run_flow_dissector()
1428 kfree(data); in bpf_prog_test_run_flow_dissector()
1438 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_sk_lookup()
1441 int ret = -EINVAL; in bpf_prog_test_run_sk_lookup()
1443 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_sk_lookup()
1444 return -EINVAL; in bpf_prog_test_run_sk_lookup()
1446 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || in bpf_prog_test_run_sk_lookup()
1447 kattr->test.data_size_out) in bpf_prog_test_run_sk_lookup()
1448 return -EINVAL; in bpf_prog_test_run_sk_lookup()
1458 return -EINVAL; in bpf_prog_test_run_sk_lookup()
1460 if (user_ctx->sk) in bpf_prog_test_run_sk_lookup()
1461 goto out; in bpf_prog_test_run_sk_lookup()
1464 goto out; in bpf_prog_test_run_sk_lookup()
1466 if (user_ctx->local_port > U16_MAX) { in bpf_prog_test_run_sk_lookup()
1467 ret = -ERANGE; in bpf_prog_test_run_sk_lookup()
1468 goto out; in bpf_prog_test_run_sk_lookup()
1471 ctx.family = (u16)user_ctx->family; in bpf_prog_test_run_sk_lookup()
1472 ctx.protocol = (u16)user_ctx->protocol; in bpf_prog_test_run_sk_lookup()
1473 ctx.dport = (u16)user_ctx->local_port; in bpf_prog_test_run_sk_lookup()
1474 ctx.sport = user_ctx->remote_port; in bpf_prog_test_run_sk_lookup()
1478 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; in bpf_prog_test_run_sk_lookup()
1479 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; in bpf_prog_test_run_sk_lookup()
1484 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; in bpf_prog_test_run_sk_lookup()
1485 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; in bpf_prog_test_run_sk_lookup()
1490 ret = -EAFNOSUPPORT; in bpf_prog_test_run_sk_lookup()
1491 goto out; in bpf_prog_test_run_sk_lookup()
1496 ret = -ENOMEM; in bpf_prog_test_run_sk_lookup()
1497 goto out; in bpf_prog_test_run_sk_lookup()
1500 progs->items[0].prog = prog; in bpf_prog_test_run_sk_lookup()
1510 goto out; in bpf_prog_test_run_sk_lookup()
1512 user_ctx->cookie = 0; in bpf_prog_test_run_sk_lookup()
1514 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { in bpf_prog_test_run_sk_lookup()
1515 ret = -EOPNOTSUPP; in bpf_prog_test_run_sk_lookup()
1516 goto out; in bpf_prog_test_run_sk_lookup()
1519 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); in bpf_prog_test_run_sk_lookup()
1526 out: in bpf_prog_test_run_sk_lookup()
1536 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_prog_test_run_syscall()
1537 __u32 ctx_size_in = kattr->test.ctx_size_in; in bpf_prog_test_run_syscall()
1542 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ in bpf_prog_test_run_syscall()
1543 if (kattr->test.data_in || kattr->test.data_out || in bpf_prog_test_run_syscall()
1544 kattr->test.ctx_out || kattr->test.duration || in bpf_prog_test_run_syscall()
1545 kattr->test.repeat || kattr->test.flags || in bpf_prog_test_run_syscall()
1546 kattr->test.batch_size) in bpf_prog_test_run_syscall()
1547 return -EINVAL; in bpf_prog_test_run_syscall()
1549 if (ctx_size_in < prog->aux->max_ctx_offset || in bpf_prog_test_run_syscall()
1551 return -EINVAL; in bpf_prog_test_run_syscall()
1563 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { in bpf_prog_test_run_syscall()
1564 err = -EFAULT; in bpf_prog_test_run_syscall()
1565 goto out; in bpf_prog_test_run_syscall()
1569 err = -EFAULT; in bpf_prog_test_run_syscall()
1570 out: in bpf_prog_test_run_syscall()
1579 if (user->in || user->out) in verify_and_copy_hook_state()
1580 return -EINVAL; in verify_and_copy_hook_state()
1582 if (user->net || user->sk || user->okfn) in verify_and_copy_hook_state()
1583 return -EINVAL; in verify_and_copy_hook_state()
1585 switch (user->pf) { in verify_and_copy_hook_state()
1588 switch (state->hook) { in verify_and_copy_hook_state()
1590 state->in = dev; in verify_and_copy_hook_state()
1593 state->in = dev; in verify_and_copy_hook_state()
1596 state->in = dev; in verify_and_copy_hook_state()
1597 state->out = dev; in verify_and_copy_hook_state()
1600 state->out = dev; in verify_and_copy_hook_state()
1603 state->out = dev; in verify_and_copy_hook_state()
1609 return -EINVAL; in verify_and_copy_hook_state()
1612 state->pf = user->pf; in verify_and_copy_hook_state()
1613 state->hook = user->hook; in verify_and_copy_hook_state()
1634 struct net *net = current->nsproxy->net_ns; in bpf_prog_test_run_nf()
1635 struct net_device *dev = net->loopback_dev; in bpf_prog_test_run_nf()
1640 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_nf()
1641 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_nf()
1647 void *data; in bpf_prog_test_run_nf() local
1650 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_nf()
1651 return -EINVAL; in bpf_prog_test_run_nf()
1654 return -EINVAL; in bpf_prog_test_run_nf()
1656 data = bpf_test_init(kattr, kattr->test.data_size_in, size, in bpf_prog_test_run_nf()
1659 if (IS_ERR(data)) in bpf_prog_test_run_nf()
1660 return PTR_ERR(data); in bpf_prog_test_run_nf()
1667 kfree(data); in bpf_prog_test_run_nf()
1674 goto out; in bpf_prog_test_run_nf()
1677 skb = slab_build_skb(data); in bpf_prog_test_run_nf()
1679 ret = -ENOMEM; in bpf_prog_test_run_nf()
1680 goto out; in bpf_prog_test_run_nf()
1683 data = NULL; /* data released via kfree_skb */ in bpf_prog_test_run_nf()
1688 ret = -EINVAL; in bpf_prog_test_run_nf()
1692 goto out; in bpf_prog_test_run_nf()
1694 skb->protocol = eth_type_trans(skb, dev); in bpf_prog_test_run_nf()
1695 switch (skb->protocol) { in bpf_prog_test_run_nf()
1699 goto out; in bpf_prog_test_run_nf()
1702 goto out; in bpf_prog_test_run_nf()
1705 goto out; in bpf_prog_test_run_nf()
1707 ret = -EPROTO; in bpf_prog_test_run_nf()
1708 goto out; in bpf_prog_test_run_nf()
1713 skb->protocol = nfproto_eth(hook_state.pf); in bpf_prog_test_run_nf()
1720 goto out; in bpf_prog_test_run_nf()
1724 out: in bpf_prog_test_run_nf()
1727 kfree(data); in bpf_prog_test_run_nf()