197 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
198 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
199 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
200 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
201 static int ref_set_non_owning(struct bpf_verifier_env *env,
203 static void specialize_kfunc(struct bpf_verifier_env *env,
356 	struct bpf_verifier_env *env = private_data;  in verbose()  local
359 	if (!bpf_verifier_log_needed(&env->log))  in verbose()
363 	bpf_verifier_vlog(&env->log, fmt, args);  in verbose()
367 static void verbose_invalid_scalar(struct bpf_verifier_env *env,  in verbose_invalid_scalar()  argument
374 	verbose(env, "%s the register %s has", ctx, reg_name);  in verbose_invalid_scalar()
376 		verbose(env, " smin=%lld", reg->smin_value);  in verbose_invalid_scalar()
380 		verbose(env, " smax=%lld", reg->smax_value);  in verbose_invalid_scalar()
384 		verbose(env, " unknown scalar value");  in verbose_invalid_scalar()
385 	verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);  in verbose_invalid_scalar()
421 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)  in subprog_is_global()  argument
423 	struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;  in subprog_is_global()
428 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)  in subprog_name()  argument
432 	if (!env->prog->aux->func_info)  in subprog_name()
435 	info = &env->prog->aux->func_info[subprog];  in subprog_name()
436 	return btf_type_name(env->prog->aux->btf, info->type_id);  in subprog_name()
439 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)  in mark_subprog_exc_cb()  argument
441 	struct bpf_subprog_info *info = subprog_info(env, subprog);  in mark_subprog_exc_cb()
448 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)  in subprog_is_exc_cb()  argument
450 	return subprog_info(env, subprog)->is_exception_cb;  in subprog_is_exc_cb()
543 static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)  in is_may_goto_insn_at()  argument
545 	return is_may_goto_insn(&env->prog->insnsi[insn_idx]);  in is_may_goto_insn_at()
583 static struct bpf_func_state *func(struct bpf_verifier_env *env,  in func()  argument
586 	struct bpf_verifier_state *cur = env->cur_state;  in func()
605 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in stack_slot_obj_get_spi()  argument
611 		verbose(env, "%s has to be at a constant offset\n", obj_kind);  in stack_slot_obj_get_spi()
617 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);  in stack_slot_obj_get_spi()
623 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);  in stack_slot_obj_get_spi()
627 	if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots))  in stack_slot_obj_get_spi()
632 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in dynptr_get_spi()  argument
634 	return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);  in dynptr_get_spi()
637 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)  in iter_get_spi()  argument
639 	return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);  in iter_get_spi()
683 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
686 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,  in mark_dynptr_stack_regs()  argument
691 	int id = ++env->id_gen;  in mark_dynptr_stack_regs()
697 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,  in mark_dynptr_cb_reg()  argument
701 	__mark_dynptr_reg(reg, type, true, ++env->id_gen);  in mark_dynptr_cb_reg()
704 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
707 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in mark_stack_slots_dynptr()  argument
710 	struct bpf_func_state *state = func(env, reg);  in mark_stack_slots_dynptr()
714 	spi = dynptr_get_spi(env, reg);  in mark_stack_slots_dynptr()
727 	err = destroy_if_dynptr_stack_slot(env, state, spi);  in mark_stack_slots_dynptr()
730 	err = destroy_if_dynptr_stack_slot(env, state, spi - 1);  in mark_stack_slots_dynptr()
743 	mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,  in mark_stack_slots_dynptr()
753 			id = acquire_reference_state(env, insn_idx);  in mark_stack_slots_dynptr()
768 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi)  in invalidate_dynptr()  argument
777 	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);  in invalidate_dynptr()
778 	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);  in invalidate_dynptr()
805 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in unmark_stack_slots_dynptr()  argument
807 	struct bpf_func_state *state = func(env, reg);  in unmark_stack_slots_dynptr()
810 	spi = dynptr_get_spi(env, reg);  in unmark_stack_slots_dynptr()
815 		invalidate_dynptr(env, state, spi);  in unmark_stack_slots_dynptr()
829 	WARN_ON_ONCE(release_reference(env, ref_obj_id));  in unmark_stack_slots_dynptr()
841 			verbose(env, "verifier internal error: misconfigured ref_obj_id\n");  in unmark_stack_slots_dynptr()
845 			invalidate_dynptr(env, state, i);  in unmark_stack_slots_dynptr()
851 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
854 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in mark_reg_invalid()  argument
856 	if (!env->allow_ptr_leaks)  in mark_reg_invalid()
857 		__mark_reg_not_init(env, reg);  in mark_reg_invalid()
859 		__mark_reg_unknown(env, reg);  in mark_reg_invalid()
862 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,  in destroy_if_dynptr_stack_slot()  argument
882 		verbose(env, "cannot overwrite referenced dynptr\n");  in destroy_if_dynptr_stack_slot()
886 	mark_stack_slot_scratched(env, spi);  in destroy_if_dynptr_stack_slot()
887 	mark_stack_slot_scratched(env, spi - 1);  in destroy_if_dynptr_stack_slot()
897 	bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({  in destroy_if_dynptr_stack_slot()
902 			mark_reg_invalid(env, dreg);  in destroy_if_dynptr_stack_slot()
908 	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);  in destroy_if_dynptr_stack_slot()
909 	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);  in destroy_if_dynptr_stack_slot()
918 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in is_dynptr_reg_valid_uninit()  argument
925 	spi = dynptr_get_spi(env, reg);  in is_dynptr_reg_valid_uninit()
946 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in is_dynptr_reg_valid_init()  argument
948 	struct bpf_func_state *state = func(env, reg);  in is_dynptr_reg_valid_init()
960 	spi = dynptr_get_spi(env, reg);  in is_dynptr_reg_valid_init()
975 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in is_dynptr_type_expected()  argument
978 	struct bpf_func_state *state = func(env, reg);  in is_dynptr_type_expected()
990 		spi = dynptr_get_spi(env, reg);  in is_dynptr_type_expected()
999 static bool in_rcu_cs(struct bpf_verifier_env *env);
1003 static int mark_stack_slots_iter(struct bpf_verifier_env *env,  in mark_stack_slots_iter()  argument
1008 	struct bpf_func_state *state = func(env, reg);  in mark_stack_slots_iter()
1011 	spi = iter_get_spi(env, reg, nr_slots);  in mark_stack_slots_iter()
1015 	id = acquire_reference_state(env, insn_idx);  in mark_stack_slots_iter()
1026 			if (in_rcu_cs(env))  in mark_stack_slots_iter()
1041 		mark_stack_slot_scratched(env, spi - i);  in mark_stack_slots_iter()
1047 static int unmark_stack_slots_iter(struct bpf_verifier_env *env,  in unmark_stack_slots_iter()  argument
1050 	struct bpf_func_state *state = func(env, reg);  in unmark_stack_slots_iter()
1053 	spi = iter_get_spi(env, reg, nr_slots);  in unmark_stack_slots_iter()
1062 			WARN_ON_ONCE(release_reference(env, st->ref_obj_id));  in unmark_stack_slots_iter()
1064 		__mark_reg_not_init(env, st);  in unmark_stack_slots_iter()
1072 		mark_stack_slot_scratched(env, spi - i);  in unmark_stack_slots_iter()
1078 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,  in is_iter_reg_valid_uninit()  argument
1081 	struct bpf_func_state *state = func(env, reg);  in is_iter_reg_valid_uninit()
1088 	spi = iter_get_spi(env, reg, nr_slots);  in is_iter_reg_valid_uninit()
1105 static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in is_iter_reg_valid_init()  argument
1108 	struct bpf_func_state *state = func(env, reg);  in is_iter_reg_valid_init()
1111 	spi = iter_get_spi(env, reg, nr_slots);  in is_iter_reg_valid_init()
1185  * env->allow_ptr_leaks into account and force STACK_MISC, if necessary.
1187 static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)  in mark_stack_slot_misc()  argument
1191 	if (env->allow_ptr_leaks && *stype == STACK_INVALID)  in mark_stack_slot_misc()
1299 static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)  in grow_stack_state()  argument
1317 	if (env->subprog_info[state->subprogno].stack_depth < size)  in grow_stack_state()
1318 		env->subprog_info[state->subprogno].stack_depth = size;  in grow_stack_state()
1323 /* Acquire a pointer id from the env and update the state->refs to include
1328 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)  in acquire_reference_state()  argument
1330 	struct bpf_func_state *state = cur_func(env);  in acquire_reference_state()
1337 	id = ++env->id_gen;  in acquire_reference_state()
1462 static u32 state_htab_size(struct bpf_verifier_env *env)  in state_htab_size()  argument
1464 	return env->prog->len;  in state_htab_size()
1467 static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx)  in explored_state()  argument
1469 	struct bpf_verifier_state *cur = env->cur_state;  in explored_state()
1472 	return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];  in explored_state()
1646 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)  in update_branch_counts()  argument
1671 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,  in pop_stack()  argument
1674 	struct bpf_verifier_state *cur = env->cur_state;  in pop_stack()
1675 	struct bpf_verifier_stack_elem *elem, *head = env->head;  in pop_stack()
1678 	if (env->head == NULL)  in pop_stack()
1687 		bpf_vlog_reset(&env->log, head->log_pos);  in pop_stack()
1695 	env->head = elem;  in pop_stack()
1696 	env->stack_size--;  in pop_stack()
1700 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,  in push_stack()  argument
1704 	struct bpf_verifier_state *cur = env->cur_state;  in push_stack()
1714 	elem->next = env->head;  in push_stack()
1715 	elem->log_pos = env->log.end_pos;  in push_stack()
1716 	env->head = elem;  in push_stack()
1717 	env->stack_size++;  in push_stack()
1722 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {  in push_stack()
1723 		verbose(env, "The sequence of %d jumps is too complex.\n",  in push_stack()
1724 			env->stack_size);  in push_stack()
1741 	free_verifier_state(env->cur_state, true);  in push_stack()
1742 	env->cur_state = NULL;  in push_stack()
1744 	while (!pop_stack(env, NULL, NULL, false));  in push_stack()
1798 static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in __mark_reg_const_zero()  argument
1805 	reg->precise = !env->bpf_capable;  in __mark_reg_const_zero()
1808 static void mark_reg_known_zero(struct bpf_verifier_env *env,  in mark_reg_known_zero()  argument
1812 		verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);  in mark_reg_known_zero()
1815 			__mark_reg_not_init(env, regs + regno);  in mark_reg_known_zero()
2258 static int reg_bounds_sanity_check(struct bpf_verifier_env *env,  in reg_bounds_sanity_check()  argument
2295 	verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] "  in reg_bounds_sanity_check()
2302 	if (env->test_reg_invariants)  in reg_bounds_sanity_check()
2352 static void __mark_reg_unknown(const struct bpf_verifier_env *env,  in __mark_reg_unknown()  argument
2356 	reg->precise = !env->bpf_capable;  in __mark_reg_unknown()
2359 static void mark_reg_unknown(struct bpf_verifier_env *env,  in mark_reg_unknown()  argument
2363 		verbose(env, "mark_reg_unknown(regs, %u)\n", regno);  in mark_reg_unknown()
2366 			__mark_reg_not_init(env, regs + regno);  in mark_reg_unknown()
2369 	__mark_reg_unknown(env, regs + regno);  in mark_reg_unknown()
2372 static int __mark_reg_s32_range(struct bpf_verifier_env *env,  in __mark_reg_s32_range()  argument
2388 	return reg_bounds_sanity_check(env, reg, "s32_range");  in __mark_reg_s32_range()
2391 static void __mark_reg_not_init(const struct bpf_verifier_env *env,  in __mark_reg_not_init()  argument
2394 	__mark_reg_unknown(env, reg);  in __mark_reg_not_init()
2398 static void mark_reg_not_init(struct bpf_verifier_env *env,  in mark_reg_not_init()  argument
2402 		verbose(env, "mark_reg_not_init(regs, %u)\n", regno);  in mark_reg_not_init()
2405 			__mark_reg_not_init(env, regs + regno);  in mark_reg_not_init()
2408 	__mark_reg_not_init(env, regs + regno);  in mark_reg_not_init()
2411 static void mark_btf_ld_reg(struct bpf_verifier_env *env,  in mark_btf_ld_reg()  argument
2418 		mark_reg_unknown(env, regs, regno);  in mark_btf_ld_reg()
2421 	mark_reg_known_zero(env, regs, regno);  in mark_btf_ld_reg()
2426 		regs[regno].id = ++env->id_gen;  in mark_btf_ld_reg()
2430 static void init_reg_state(struct bpf_verifier_env *env,  in init_reg_state()  argument
2437 		mark_reg_not_init(env, regs, i);  in init_reg_state()
2445 	mark_reg_known_zero(env, regs, BPF_REG_FP);  in init_reg_state()
2455 static void init_func_state(struct bpf_verifier_env *env,  in init_func_state()  argument
2463 	init_reg_state(env, state);  in init_func_state()
2464 	mark_verifier_state_scratched(env);  in init_func_state()
2468 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,  in push_async_cb()  argument
2481 	elem->next = env->head;  in push_async_cb()
2482 	elem->log_pos = env->log.end_pos;  in push_async_cb()
2483 	env->head = elem;  in push_async_cb()
2484 	env->stack_size++;  in push_async_cb()
2485 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {  in push_async_cb()
2486 		verbose(env,  in push_async_cb()
2488 			env->stack_size);  in push_async_cb()
2501 	init_func_state(env, frame,  in push_async_cb()
2508 	free_verifier_state(env->cur_state, true);  in push_async_cb()
2509 	env->cur_state = NULL;  in push_async_cb()
2511 	while (!pop_stack(env, NULL, NULL, false));  in push_async_cb()
2528 static int find_subprog(struct bpf_verifier_env *env, int off)  in find_subprog()  argument
2532 	p = bsearch(&off, env->subprog_info, env->subprog_cnt,  in find_subprog()
2533 		    sizeof(env->subprog_info[0]), cmp_subprogs);  in find_subprog()
2536 	return p - env->subprog_info;  in find_subprog()
2540 static int add_subprog(struct bpf_verifier_env *env, int off)  in add_subprog()  argument
2542 	int insn_cnt = env->prog->len;  in add_subprog()
2546 		verbose(env, "call to invalid destination\n");  in add_subprog()
2549 	ret = find_subprog(env, off);  in add_subprog()
2552 	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {  in add_subprog()
2553 		verbose(env, "too many subprograms\n");  in add_subprog()
2557 	env->subprog_info[env->subprog_cnt++].start = off;  in add_subprog()
2558 	sort(env->subprog_info, env->subprog_cnt,  in add_subprog()
2559 	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);  in add_subprog()
2560 	return env->subprog_cnt - 1;  in add_subprog()
2563 static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)  in bpf_find_exception_callback_insn_off()  argument
2565 	struct bpf_prog_aux *aux = env->prog->aux;  in bpf_find_exception_callback_insn_off()
2579 		verbose(env, "invalid btf id for main subprog in func_info\n");  in bpf_find_exception_callback_insn_off()
2590 			verbose(env, "multiple exception callback tags for main subprog\n");  in bpf_find_exception_callback_insn_off()
2596 		verbose(env, "exception callback '%s' could not be found in BTF\n", name);  in bpf_find_exception_callback_insn_off()
2602 		verbose(env, "exception callback '%s' must have global linkage\n", name);  in bpf_find_exception_callback_insn_off()
2614 			verbose(env, "invalid exception callback insn_off in func_info: 0\n");  in bpf_find_exception_callback_insn_off()
2619 		verbose(env, "exception callback type id not found in func_info\n");  in bpf_find_exception_callback_insn_off()
2701 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,  in __find_kfunc_desc_btf()  argument
2711 	tab = env->prog->aux->kfunc_btf_tab;  in __find_kfunc_desc_btf()
2716 			verbose(env, "too many different module BTFs\n");  in __find_kfunc_desc_btf()
2720 		if (bpfptr_is_null(env->fd_array)) {  in __find_kfunc_desc_btf()
2721 			verbose(env, "kfunc offset > 0 without fd_array is invalid\n");  in __find_kfunc_desc_btf()
2725 		if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,  in __find_kfunc_desc_btf()
2732 			verbose(env, "invalid module BTF fd specified\n");  in __find_kfunc_desc_btf()
2737 			verbose(env, "BTF fd for kfunc is not a module BTF\n");  in __find_kfunc_desc_btf()
2777 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)  in find_kfunc_desc_btf()  argument
2784 			verbose(env, "negative offset disallowed for kernel module function call\n");  in find_kfunc_desc_btf()
2788 		return __find_kfunc_desc_btf(env, offset);  in find_kfunc_desc_btf()
2793 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)  in add_kfunc_call()  argument
2806 	prog_aux = env->prog->aux;  in add_kfunc_call()
2811 			verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");  in add_kfunc_call()
2815 		if (!env->prog->jit_requested) {  in add_kfunc_call()
2816 			verbose(env, "JIT is required for calling kernel function\n");  in add_kfunc_call()
2821 			verbose(env, "JIT does not support calling kernel function\n");  in add_kfunc_call()
2825 		if (!env->prog->gpl_compatible) {  in add_kfunc_call()
2826 			verbose(env, "cannot call kernel function from non-GPL compatible program\n");  in add_kfunc_call()
2852 	desc_btf = find_kfunc_desc_btf(env, offset);  in add_kfunc_call()
2854 		verbose(env, "failed to find BTF for kernel function\n");  in add_kfunc_call()
2858 	if (find_kfunc_desc(env->prog, func_id, offset))  in add_kfunc_call()
2862 		verbose(env, "too many different kernel function calls\n");  in add_kfunc_call()
2868 		verbose(env, "kernel btf_id %u is not a function\n",  in add_kfunc_call()
2874 		verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",  in add_kfunc_call()
2882 		verbose(env, "cannot find address for kernel function %s\n",  in add_kfunc_call()
2886 	specialize_kfunc(env, func_id, offset, &addr);  in add_kfunc_call()
2894 			verbose(env, "address of kernel function %s is out of range\n",  in add_kfunc_call()
2901 		err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);  in add_kfunc_call()
2911 	err = btf_distill_func_proto(&env->log, desc_btf,  in add_kfunc_call()
2967 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)  in add_subprog_and_kfunc()  argument
2969 	struct bpf_subprog_info *subprog = env->subprog_info;  in add_subprog_and_kfunc()
2970 	int i, ret, insn_cnt = env->prog->len, ex_cb_insn;  in add_subprog_and_kfunc()
2971 	struct bpf_insn *insn = env->prog->insnsi;  in add_subprog_and_kfunc()
2974 	ret = add_subprog(env, 0);  in add_subprog_and_kfunc()
2983 		if (!env->bpf_capable) {  in add_subprog_and_kfunc()
2984 …verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_AD…  in add_subprog_and_kfunc()
2989 			ret = add_subprog(env, i + insn->imm + 1);  in add_subprog_and_kfunc()
2991 			ret = add_kfunc_call(env, insn->imm, insn->off);  in add_subprog_and_kfunc()
2997 	ret = bpf_find_exception_callback_insn_off(env);  in add_subprog_and_kfunc()
3006 		ret = add_subprog(env, ex_cb_insn);  in add_subprog_and_kfunc()
3009 		for (i = 1; i < env->subprog_cnt; i++) {  in add_subprog_and_kfunc()
3010 			if (env->subprog_info[i].start != ex_cb_insn)  in add_subprog_and_kfunc()
3012 			env->exception_callback_subprog = i;  in add_subprog_and_kfunc()
3013 			mark_subprog_exc_cb(env, i);  in add_subprog_and_kfunc()
3021 	subprog[env->subprog_cnt].start = insn_cnt;  in add_subprog_and_kfunc()
3023 	if (env->log.level & BPF_LOG_LEVEL2)  in add_subprog_and_kfunc()
3024 		for (i = 0; i < env->subprog_cnt; i++)  in add_subprog_and_kfunc()
3025 			verbose(env, "func#%d @%d\n", i, subprog[i].start);  in add_subprog_and_kfunc()
3030 static int check_subprogs(struct bpf_verifier_env *env)  in check_subprogs()  argument
3033 	struct bpf_subprog_info *subprog = env->subprog_info;  in check_subprogs()
3034 	struct bpf_insn *insn = env->prog->insnsi;  in check_subprogs()
3035 	int insn_cnt = env->prog->len;  in check_subprogs()
3061 			verbose(env, "jump out of range from insn %d to %d\n", i, off);  in check_subprogs()
3073 				verbose(env, "last insn is not an exit or jmp\n");  in check_subprogs()
3078 			if (cur_subprog < env->subprog_cnt)  in check_subprogs()
3088 static int mark_reg_read(struct bpf_verifier_env *env,  in mark_reg_read()  argument
3100 			verbose(env, "verifier BUG type %s var_off %lld off %d\n",  in mark_reg_read()
3101 				reg_type_str(env, parent->type),  in mark_reg_read()
3131 	if (env->longest_mark_read_walk < cnt)  in mark_reg_read()
3132 		env->longest_mark_read_walk = cnt;  in mark_reg_read()
3136 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in mark_dynptr_read()  argument
3138 	struct bpf_func_state *state = func(env, reg);  in mark_dynptr_read()
3147 	spi = dynptr_get_spi(env, reg);  in mark_dynptr_read()
3154 	ret = mark_reg_read(env, &state->stack[spi].spilled_ptr,  in mark_dynptr_read()
3158 	return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr,  in mark_dynptr_read()
3162 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in mark_iter_read()  argument
3165 	struct bpf_func_state *state = func(env, reg);  in mark_iter_read()
3171 		err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64);  in mark_iter_read()
3175 		mark_stack_slot_scratched(env, spi - i);  in mark_iter_read()
3185 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,  in is_reg64()  argument
3296 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)  in insn_has_def32()  argument
3303 	return !is_reg64(env, insn, dst_reg, NULL, DST_OP);  in insn_has_def32()
3306 static void mark_insn_zext(struct bpf_verifier_env *env,  in mark_insn_zext()  argument
3314 	env->insn_aux_data[def_idx - 1].zext_dst = true;  in mark_insn_zext()
3319 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,  in __check_reg_arg()  argument
3322 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;  in __check_reg_arg()
3327 		verbose(env, "R%d is invalid\n", regno);  in __check_reg_arg()
3331 	mark_reg_scratched(env, regno);  in __check_reg_arg()
3334 	rw64 = is_reg64(env, insn, regno, reg, t);  in __check_reg_arg()
3338 			verbose(env, "R%d !read_ok\n", regno);  in __check_reg_arg()
3346 			mark_insn_zext(env, reg);  in __check_reg_arg()
3348 		return mark_reg_read(env, reg, reg->parent,  in __check_reg_arg()
3353 			verbose(env, "frame pointer is read only\n");  in __check_reg_arg()
3357 		reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;  in __check_reg_arg()
3359 			mark_reg_unknown(env, regs, regno);  in __check_reg_arg()
3364 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,  in check_reg_arg()  argument
3367 	struct bpf_verifier_state *vstate = env->cur_state;  in check_reg_arg()
3370 	return __check_reg_arg(env, state->regs, regno, t);  in check_reg_arg()
3388 static void mark_jmp_point(struct bpf_verifier_env *env, int idx)  in mark_jmp_point()  argument
3390 	env->insn_aux_data[idx].jmp_point = true;  in mark_jmp_point()
3393 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)  in is_jmp_point()  argument
3395 	return env->insn_aux_data[insn_idx].jmp_point;  in is_jmp_point()
3477 static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,  in push_jmp_history()  argument
3485 	if (env->cur_hist_ent) {  in push_jmp_history()
3489 		WARN_ONCE((env->cur_hist_ent->flags & insn_flags) &&  in push_jmp_history()
3490 			  (env->cur_hist_ent->flags & insn_flags) != insn_flags,  in push_jmp_history()
3492 			  env->insn_idx, env->cur_hist_ent->flags, insn_flags);  in push_jmp_history()
3493 		env->cur_hist_ent->flags |= insn_flags;  in push_jmp_history()
3494 		WARN_ONCE(env->cur_hist_ent->linked_regs != 0,  in push_jmp_history()
3496 			  env->insn_idx, env->cur_hist_ent->linked_regs);  in push_jmp_history()
3497 		env->cur_hist_ent->linked_regs = linked_regs;  in push_jmp_history()
3509 	p->idx = env->insn_idx;  in push_jmp_history()
3510 	p->prev_idx = env->prev_insn_idx;  in push_jmp_history()
3514 	env->cur_hist_ent = p;  in push_jmp_history()
3584 	struct bpf_verifier_env *env = bt->env;  in bt_reset()  local
3587 	bt->env = env;  in bt_reset()
3604 		verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame);  in bt_subprog_enter()
3615 		verbose(bt->env, "BUG subprog exit from frame 0\n");  in bt_subprog_exit()
3763 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);
3774 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,  in backtrack_insn()  argument
3780 		.private_data	= env,  in backtrack_insn()
3782 	struct bpf_insn *insn = env->prog->insnsi + idx;  in backtrack_insn()
3792 	if (env->log.level & BPF_LOG_LEVEL2) {  in backtrack_insn()
3793 		fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));  in backtrack_insn()
3794 		verbose(env, "mark_precise: frame%d: regs=%s ",  in backtrack_insn()
3795 			bt->frame, env->tmp_str_buf);  in backtrack_insn()
3796 		fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));  in backtrack_insn()
3797 		verbose(env, "stack=%s before ", env->tmp_str_buf);  in backtrack_insn()
3798 		verbose(env, "%d: ", idx);  in backtrack_insn()
3799 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);  in backtrack_insn()
3888 			subprog = find_subprog(env, subprog_insn_idx);  in backtrack_insn()
3892 			if (subprog_is_global(env, subprog)) {  in backtrack_insn()
3905 					verbose(env, "BUG regs %x\n", bt_reg_mask(bt));  in backtrack_insn()
3920 					verbose(env, "BUG regs %x\n", bt_reg_mask(bt));  in backtrack_insn()
3928 					verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));  in backtrack_insn()
3952 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));  in backtrack_insn()
3957 				verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));  in backtrack_insn()
3980 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));  in backtrack_insn()
3995 			if (subseq_idx >= 0 && calls_callback(env, subseq_idx))  in backtrack_insn()
3999 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));  in backtrack_insn()
4013 				     bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&  in backtrack_insn()
4115 static void mark_all_scalars_precise(struct bpf_verifier_env *env,  in mark_all_scalars_precise()  argument
4122 	if (env->log.level & BPF_LOG_LEVEL2) {  in mark_all_scalars_precise()
4123 		verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",  in mark_all_scalars_precise()
4141 				if (env->log.level & BPF_LOG_LEVEL2) {  in mark_all_scalars_precise()
4142 					verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",  in mark_all_scalars_precise()
4153 				if (env->log.level & BPF_LOG_LEVEL2) {  in mark_all_scalars_precise()
4154 					verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",  in mark_all_scalars_precise()
4162 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)  in mark_all_scalars_imprecise()  argument
4204  * i.e., it is not yet put into env->explored_states, and it has no children
4207  * reached or b) checkpointed and put into env->explored_states, branching out
4274 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno)  in __mark_chain_precision()  argument
4276 	struct backtrack_state *bt = &env->bt;  in __mark_chain_precision()
4277 	struct bpf_verifier_state *st = env->cur_state;  in __mark_chain_precision()
4279 	int last_idx = env->insn_idx;  in __mark_chain_precision()
4286 	if (!env->bpf_capable)  in __mark_chain_precision()
4290 	bt_init(bt, env->cur_state->curframe);  in __mark_chain_precision()
4314 		if (env->log.level & BPF_LOG_LEVEL2) {  in __mark_chain_precision()
4315 			verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",  in __mark_chain_precision()
4340 			verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n",  in __mark_chain_precision()
4352 				err = backtrack_insn(env, i, subseq_idx, hist, bt);  in __mark_chain_precision()
4355 				mark_all_scalars_precise(env, env->cur_state);  in __mark_chain_precision()
4371 			if (i >= env->prog->len) {  in __mark_chain_precision()
4378 				verbose(env, "BUG backtracking idx %d\n", i);  in __mark_chain_precision()
4405 					verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n",  in __mark_chain_precision()
4421 			if (env->log.level & BPF_LOG_LEVEL2) {  in __mark_chain_precision()
4422 				fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,  in __mark_chain_precision()
4424 				verbose(env, "mark_precise: frame%d: parent state regs=%s ",  in __mark_chain_precision()
4425 					fr, env->tmp_str_buf);  in __mark_chain_precision()
4426 				fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,  in __mark_chain_precision()
4428 				verbose(env, "stack=%s: ", env->tmp_str_buf);  in __mark_chain_precision()
4429 				print_verifier_state(env, func, true);  in __mark_chain_precision()
4446 		mark_all_scalars_precise(env, env->cur_state);  in __mark_chain_precision()
4453 int mark_chain_precision(struct bpf_verifier_env *env, int regno)  in mark_chain_precision()  argument
4455 	return __mark_chain_precision(env, regno);  in mark_chain_precision()
4458 /* mark_chain_precision_batch() assumes that env->bt is set in the caller to
4461 static int mark_chain_precision_batch(struct bpf_verifier_env *env)  in mark_chain_precision_batch()  argument
4463 	return __mark_chain_precision(env, -1);  in mark_chain_precision_batch()
4521 static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,  in assign_scalar_id_before_mov()  argument
4542 		src_reg->id = ++env->id_gen;  in assign_scalar_id_before_mov()
4556 static void save_register_state(struct bpf_verifier_env *env,  in save_register_state()  argument
4572 		mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);  in save_register_state()
4586 static void check_fastcall_stack_contract(struct bpf_verifier_env *env,  in check_fastcall_stack_contract()  argument
4589 	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];  in check_fastcall_stack_contract()
4590 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in check_fastcall_stack_contract()
4613 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,  in check_stack_write_fixed_off()  argument
4621 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];  in check_stack_write_fixed_off()
4628 	if (!env->allow_ptr_leaks &&  in check_stack_write_fixed_off()
4631 		verbose(env, "attempt to corrupt spilled pointer on stack\n");  in check_stack_write_fixed_off()
4635 	cur = env->cur_state->frame[env->cur_state->curframe];  in check_stack_write_fixed_off()
4638 	if (!env->bypass_spec_v4) {  in check_stack_write_fixed_off()
4651 			env->insn_aux_data[insn_idx].sanitize_stack_spill = true;  in check_stack_write_fixed_off()
4654 	err = destroy_if_dynptr_stack_slot(env, state, spi);  in check_stack_write_fixed_off()
4658 	check_fastcall_stack_contract(env, state, insn_idx, off);  in check_stack_write_fixed_off()
4659 	mark_stack_slot_scratched(env, spi);  in check_stack_write_fixed_off()
4660 	if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {  in check_stack_write_fixed_off()
4666 			assign_scalar_id_before_mov(env, reg);  in check_stack_write_fixed_off()
4667 		save_register_state(env, state, spi, reg, size);  in check_stack_write_fixed_off()
4672 		   env->bpf_capable) {  in check_stack_write_fixed_off()
4673 		struct bpf_reg_state *tmp_reg = &env->fake_reg[0];  in check_stack_write_fixed_off()
4678 		save_register_state(env, state, spi, tmp_reg, size);  in check_stack_write_fixed_off()
4682 			verbose_linfo(env, insn_idx, "; ");  in check_stack_write_fixed_off()
4683 			verbose(env, "invalid size of register spill\n");  in check_stack_write_fixed_off()
4687 			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");  in check_stack_write_fixed_off()
4690 		save_register_state(env, state, spi, reg, size);  in check_stack_write_fixed_off()
4721 			err = mark_chain_precision(env, value_regno);  in check_stack_write_fixed_off()
4734 		return push_jmp_history(env, env->cur_state, insn_flags, 0);  in check_stack_write_fixed_off()
4757 static int check_stack_write_var_off(struct bpf_verifier_env *env,  in check_stack_write_var_off()  argument
4767 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];  in check_stack_write_var_off()
4774 	cur = env->cur_state->frame[env->cur_state->curframe];  in check_stack_write_var_off()
4788 		err = destroy_if_dynptr_stack_slot(env, state, spi);  in check_stack_write_var_off()
4793 	check_fastcall_stack_contract(env, state, insn_idx, min_off);  in check_stack_write_var_off()
4802 		mark_stack_slot_scratched(env, spi);  in check_stack_write_var_off()
4804 		if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {  in check_stack_write_var_off()
4816 			verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",  in check_stack_write_var_off()
4851 		if (*stype == STACK_INVALID && !env->allow_uninit_stack) {  in check_stack_write_var_off()
4852 			verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",  in check_stack_write_var_off()
4860 		err = mark_chain_precision(env, value_regno);  in check_stack_write_var_off()
4875 static void mark_reg_stack_read(struct bpf_verifier_env *env,  in mark_reg_stack_read()  argument
4880 	struct bpf_verifier_state *vstate = env->cur_state;  in mark_reg_stack_read()
4889 		mark_stack_slot_scratched(env, spi);  in mark_reg_stack_read()
4899 		__mark_reg_const_zero(env, &state->regs[dst_regno]);  in mark_reg_stack_read()
4902 		mark_reg_unknown(env, state->regs, dst_regno);  in mark_reg_stack_read()
4916 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,  in check_stack_read_fixed_off()  argument
4921 	struct bpf_verifier_state *vstate = env->cur_state;  in check_stack_read_fixed_off()
4931 	mark_stack_slot_scratched(env, spi);  in check_stack_read_fixed_off()
4932 	check_fastcall_stack_contract(env, state, env->insn_idx, off);  in check_stack_read_fixed_off()
4942 				verbose_linfo(env, env->insn_idx, "; ");  in check_stack_read_fixed_off()
4943 				verbose(env, "invalid size of register fill\n");  in check_stack_read_fixed_off()
4947 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in check_stack_read_fixed_off()
4981 					if (type == STACK_INVALID && env->allow_uninit_stack)  in check_stack_read_fixed_off()
4983 					verbose(env, "invalid read from stack off %d+%d size %d\n",  in check_stack_read_fixed_off()
4990 					__mark_reg_const_zero(env, &state->regs[dst_regno]);  in check_stack_read_fixed_off()
4994 					__mark_reg_const_zero(env, &state->regs[dst_regno]);  in check_stack_read_fixed_off()
4997 					mark_reg_unknown(env, state->regs, dst_regno);  in check_stack_read_fixed_off()
5010 		} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {  in check_stack_read_fixed_off()
5017 			verbose(env, "leaking pointer from stack off %d\n",  in check_stack_read_fixed_off()
5021 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in check_stack_read_fixed_off()
5029 			if (type == STACK_INVALID && env->allow_uninit_stack)  in check_stack_read_fixed_off()
5031 			verbose(env, "invalid read from stack off %d+%d size %d\n",  in check_stack_read_fixed_off()
5035 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in check_stack_read_fixed_off()
5037 			mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);  in check_stack_read_fixed_off()
5041 		return push_jmp_history(env, env->cur_state, insn_flags, 0);  in check_stack_read_fixed_off()
5050 static int check_stack_range_initialized(struct bpf_verifier_env *env,
5056 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)  in reg_state()  argument
5058 	return cur_regs(env) + regno;  in reg_state()
5074 static int check_stack_read_var_off(struct bpf_verifier_env *env,  in check_stack_read_var_off()  argument
5078 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read_var_off()
5079 	struct bpf_func_state *ptr_state = func(env, reg);  in check_stack_read_var_off()
5085 	err = check_stack_range_initialized(env, ptr_regno, off, size,  in check_stack_read_var_off()
5092 	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);  in check_stack_read_var_off()
5093 	check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);  in check_stack_read_var_off()
5106 static int check_stack_read(struct bpf_verifier_env *env,  in check_stack_read()  argument
5110 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_read()
5111 	struct bpf_func_state *state = func(env, reg);  in check_stack_read()
5124 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=…  in check_stack_read()
5139 		err = check_stack_read_fixed_off(env, state, off, size,  in check_stack_read()
5146 		err = check_stack_read_var_off(env, ptr_regno, off, size,  in check_stack_read()
5163 static int check_stack_write(struct bpf_verifier_env *env,  in check_stack_write()  argument
5167 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);  in check_stack_write()
5168 	struct bpf_func_state *state = func(env, reg);  in check_stack_write()
5173 		err = check_stack_write_fixed_off(env, state, off, size,  in check_stack_write()
5179 		err = check_stack_write_var_off(env, state,  in check_stack_write()
5186 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,  in check_map_access_type()  argument
5189 	struct bpf_reg_state *regs = cur_regs(env);  in check_map_access_type()
5194 		verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",  in check_map_access_type()
5200 		verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",  in check_map_access_type()
5209 static int __check_mem_access(struct bpf_verifier_env *env, int regno,  in __check_mem_access()  argument
5219 	reg = &cur_regs(env)[regno];  in __check_mem_access()
5222 		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",  in __check_mem_access()
5226 		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",  in __check_mem_access()
5232 		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",  in __check_mem_access()
5237 		verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",  in __check_mem_access()
5245 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,  in check_mem_region_access()  argument
5249 	struct bpf_verifier_state *vstate = env->cur_state;  in check_mem_region_access()
5268 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n…  in check_mem_region_access()
5272 	err = __check_mem_access(env, regno, reg->smin_value + off, size,  in check_mem_region_access()
5275 		verbose(env, "R%d min value is outside of the allowed memory range\n",  in check_mem_region_access()
5285 		verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",  in check_mem_region_access()
5289 	err = __check_mem_access(env, regno, reg->umax_value + off, size,  in check_mem_region_access()
5292 		verbose(env, "R%d max value is outside of the allowed memory range\n",  in check_mem_region_access()
5300 static int __check_ptr_off_reg(struct bpf_verifier_env *env,  in __check_ptr_off_reg()  argument
5309 		verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",  in __check_ptr_off_reg()
5310 			reg_type_str(env, reg->type), regno, reg->off);  in __check_ptr_off_reg()
5315 		verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",  in __check_ptr_off_reg()
5316 			reg_type_str(env, reg->type), regno, reg->off);  in __check_ptr_off_reg()
5324 		verbose(env, "variable %s access var_off=%s disallowed\n",  in __check_ptr_off_reg()
5325 			reg_type_str(env, reg->type), tn_buf);  in __check_ptr_off_reg()
5332 static int check_ptr_off_reg(struct bpf_verifier_env *env,  in check_ptr_off_reg()  argument
5335 	return __check_ptr_off_reg(env, reg, regno, false);  in check_ptr_off_reg()
5338 static int map_kptr_match_type(struct bpf_verifier_env *env,  in map_kptr_match_type()  argument
5370 	if (__check_ptr_off_reg(env, reg, regno, true))  in map_kptr_match_type()
5397 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,  in map_kptr_match_type()
5403 	verbose(env, "invalid kptr access, R%d type=%s%s ", regno,  in map_kptr_match_type()
5404 		reg_type_str(env, reg->type), reg_name);  in map_kptr_match_type()
5405 	verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);  in map_kptr_match_type()
5407 		verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),  in map_kptr_match_type()
5410 		verbose(env, "\n");  in map_kptr_match_type()
5414 static bool in_sleepable(struct bpf_verifier_env *env)  in in_sleepable()  argument
5416 	return env->prog->sleepable ||  in in_sleepable()
5417 	       (env->cur_state && env->cur_state->in_sleepable);  in in_sleepable()
5423 static bool in_rcu_cs(struct bpf_verifier_env *env)  in in_rcu_cs()  argument
5425 	return env->cur_state->active_rcu_lock ||  in in_rcu_cs()
5426 	       env->cur_state->active_lock.ptr ||  in in_rcu_cs()
5427 	       !in_sleepable(env);  in in_rcu_cs()
5471 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)  in btf_ld_kptr_type()  argument
5477 	if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {  in btf_ld_kptr_type()
5494 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,  in check_map_kptr_access()  argument
5498 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];  in check_map_kptr_access()
5510 		verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");  in check_map_kptr_access()
5519 		verbose(env, "store to referenced kptr disallowed\n");  in check_map_kptr_access()
5524 		val_reg = reg_state(env, value_regno);  in check_map_kptr_access()
5528 		mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf,  in check_map_kptr_access()
5529 				kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field));  in check_map_kptr_access()
5531 		val_reg = reg_state(env, value_regno);  in check_map_kptr_access()
5533 		    map_kptr_match_type(env, kptr_field, val_reg, value_regno))  in check_map_kptr_access()
5537 			verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",  in check_map_kptr_access()
5542 		verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");  in check_map_kptr_access()
5549 static int check_map_access(struct bpf_verifier_env *env, u32 regno,  in check_map_access()  argument
5553 	struct bpf_verifier_state *vstate = env->cur_state;  in check_map_access()
5560 	err = check_mem_region_access(env, regno, off, size, map->value_size,  in check_map_access()
5583 					verbose(env, "kptr cannot be accessed indirectly by helper\n");  in check_map_access()
5587 					verbose(env, "kptr access cannot have variable offset\n");  in check_map_access()
5591 					verbose(env, "kptr access misaligned expected=%u off=%llu\n",  in check_map_access()
5596 					verbose(env, "kptr access size must be BPF_DW\n");  in check_map_access()
5601 				verbose(env, "%s cannot be accessed directly by load/store\n",  in check_map_access()
5612 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,  in may_access_direct_pkt_data()  argument
5616 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);  in may_access_direct_pkt_data()
5640 		env->seen_direct_write = true;  in may_access_direct_pkt_data()
5645 			env->seen_direct_write = true;  in may_access_direct_pkt_data()
5654 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,  in check_packet_access()  argument
5657 	struct bpf_reg_state *regs = cur_regs(env);  in check_packet_access()
5670 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n…  in check_packet_access()
5676 	      __check_mem_access(env, regno, off, size, reg->range,  in check_packet_access()
5679 		verbose(env, "R%d offset is outside of the packet\n", regno);  in check_packet_access()
5689 	env->prog->aux->max_pkt_offset =  in check_packet_access()
5690 		max_t(u32, env->prog->aux->max_pkt_offset,  in check_packet_access()
5697 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,  in check_ctx_access()  argument
5703 		.log = &env->log,  in check_ctx_access()
5708 	if (env->ops->is_valid_access &&  in check_ctx_access()
5709 	    env->ops->is_valid_access(off, size, t, env->prog, &info)) {  in check_ctx_access()
5724 			env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;  in check_ctx_access()
5727 		if (env->prog->aux->max_ctx_offset < off + size)  in check_ctx_access()
5728 			env->prog->aux->max_ctx_offset = off + size;  in check_ctx_access()
5732 	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);  in check_ctx_access()
5736 static int check_flow_keys_access(struct bpf_verifier_env *env, int off,  in check_flow_keys_access()  argument
5741 		verbose(env, "invalid access to flow keys off=%d size=%d\n",  in check_flow_keys_access()
5748 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,  in check_sock_access()  argument
5752 	struct bpf_reg_state *regs = cur_regs(env);  in check_sock_access()
5758 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n…  in check_sock_access()
5782 		env->insn_aux_data[insn_idx].ctx_field_size =  in check_sock_access()
5787 	verbose(env, "R%d invalid %s access off=%d size=%d\n",  in check_sock_access()
5788 		regno, reg_type_str(env, reg->type), off, size);  in check_sock_access()
5793 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)  in is_pointer_value()  argument
5795 	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));  in is_pointer_value()
5798 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)  in is_ctx_reg()  argument
5800 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_ctx_reg()
5805 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)  in is_sk_reg()  argument
5807 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_sk_reg()
5812 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)  in is_pkt_reg()  argument
5814 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_pkt_reg()
5819 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)  in is_flow_key_reg()  argument
5821 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_flow_key_reg()
5827 static bool is_arena_reg(struct bpf_verifier_env *env, int regno)  in is_arena_reg()  argument
5829 	const struct bpf_reg_state *reg = reg_state(env, regno);  in is_arena_reg()
5877 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,  in check_pkt_ptr_alignment()  argument
5903 		verbose(env,  in check_pkt_ptr_alignment()
5912 static int check_generic_ptr_alignment(struct bpf_verifier_env *env,  in check_generic_ptr_alignment()  argument
5928 		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",  in check_generic_ptr_alignment()
5936 static int check_ptr_alignment(struct bpf_verifier_env *env,  in check_ptr_alignment()  argument
5940 	bool strict = env->strict_alignment || strict_alignment_once;  in check_ptr_alignment()
5949 		return check_pkt_ptr_alignment(env, reg, off, size, strict);  in check_ptr_alignment()
5987 	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,  in check_ptr_alignment()
5991 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)  in round_up_stack_depth()  argument
5993 	if (env->prog->jit_requested)  in round_up_stack_depth()
6008 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx)  in check_max_stack_depth_subprog()  argument
6010 	struct bpf_subprog_info *subprog = env->subprog_info;  in check_max_stack_depth_subprog()
6011 	struct bpf_insn *insn = env->prog->insnsi;  in check_max_stack_depth_subprog()
6040 		verbose(env,  in check_max_stack_depth_subprog()
6045 	depth += round_up_stack_depth(env, subprog[idx].stack_depth);  in check_max_stack_depth_subprog()
6047 		verbose(env, "combined stack size of %d calls is %d. Too large\n",  in check_max_stack_depth_subprog()
6071 			verbose(env,  in check_max_stack_depth_subprog()
6085 		sidx = find_subprog(env, next_insn);  in check_max_stack_depth_subprog()
6093 				verbose(env, "verifier bug. subprog has tail_call and async cb\n");  in check_max_stack_depth_subprog()
6100 				verbose(env, "insn %d cannot call exception cb directly\n", i);  in check_max_stack_depth_subprog()
6112 			verbose(env, "the call stack of %d frames is too deep !\n",  in check_max_stack_depth_subprog()
6126 				verbose(env, "cannot tail call within exception cb\n");  in check_max_stack_depth_subprog()
6132 		env->prog->aux->tail_call_reachable = true;  in check_max_stack_depth_subprog()
6139 	depth -= round_up_stack_depth(env, subprog[idx].stack_depth);  in check_max_stack_depth_subprog()
6146 static int check_max_stack_depth(struct bpf_verifier_env *env)  in check_max_stack_depth()  argument
6148 	struct bpf_subprog_info *si = env->subprog_info;  in check_max_stack_depth()
6151 	for (int i = 0; i < env->subprog_cnt; i++) {  in check_max_stack_depth()
6153 			ret = check_max_stack_depth_subprog(env, i);  in check_max_stack_depth()
6163 static int get_callee_stack_depth(struct bpf_verifier_env *env,  in get_callee_stack_depth()  argument
6168 	subprog = find_subprog(env, start);  in get_callee_stack_depth()
6174 	return env->subprog_info[subprog].stack_depth;  in get_callee_stack_depth()
6178 static int __check_buffer_access(struct bpf_verifier_env *env,  in __check_buffer_access()  argument
6184 		verbose(env,  in __check_buffer_access()
6193 		verbose(env,  in __check_buffer_access()
6202 static int check_tp_buffer_access(struct bpf_verifier_env *env,  in check_tp_buffer_access()  argument
6208 	err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);  in check_tp_buffer_access()
6212 	if (off + size > env->prog->aux->max_tp_access)  in check_tp_buffer_access()
6213 		env->prog->aux->max_tp_access = off + size;  in check_tp_buffer_access()
6218 static int check_buffer_access(struct bpf_verifier_env *env,  in check_buffer_access()  argument
6227 	err = __check_buffer_access(env, buf_info, reg, regno, off, size);  in check_buffer_access()
6541 static bool type_is_rcu(struct bpf_verifier_env *env,  in type_is_rcu()  argument
6549 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");  in type_is_rcu()
6552 static bool type_is_rcu_or_null(struct bpf_verifier_env *env,  in type_is_rcu_or_null()  argument
6560 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");  in type_is_rcu_or_null()
6563 static bool type_is_trusted(struct bpf_verifier_env *env,  in type_is_trusted()  argument
6573 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");  in type_is_trusted()
6576 static bool type_is_trusted_or_null(struct bpf_verifier_env *env,  in type_is_trusted_or_null()  argument
6582 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,  in type_is_trusted_or_null()
6586 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,  in check_ptr_to_btf_access()  argument
6600 	if (!env->allow_ptr_leaks) {  in check_ptr_to_btf_access()
6601 		verbose(env,  in check_ptr_to_btf_access()
6606 	if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {  in check_ptr_to_btf_access()
6607 		verbose(env,  in check_ptr_to_btf_access()
6613 		verbose(env,  in check_ptr_to_btf_access()
6622 		verbose(env,  in check_ptr_to_btf_access()
6629 		verbose(env,  in check_ptr_to_btf_access()
6636 		verbose(env,  in check_ptr_to_btf_access()
6642 	if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {  in check_ptr_to_btf_access()
6644 			verbose(env, "verifier internal error: reg->btf must be kernel btf\n");  in check_ptr_to_btf_access()
6647 		ret = env->ops->btf_struct_access(&env->log, reg, off, size);  in check_ptr_to_btf_access()
6654 			verbose(env, "only read is supported\n");  in check_ptr_to_btf_access()
6660 			verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n");  in check_ptr_to_btf_access()
6664 		ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);  in check_ptr_to_btf_access()
6692 		if (type_is_trusted(env, reg, field_name, btf_id)) {  in check_ptr_to_btf_access()
6694 		} else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {  in check_ptr_to_btf_access()
6696 		} else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {  in check_ptr_to_btf_access()
6697 			if (type_is_rcu(env, reg, field_name, btf_id)) {  in check_ptr_to_btf_access()
6701 				   type_is_rcu_or_null(env, reg, field_name, btf_id)) {  in check_ptr_to_btf_access()
6706 				if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&  in check_ptr_to_btf_access()
6731 		mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);  in check_ptr_to_btf_access()
6736 static int check_ptr_to_map_access(struct bpf_verifier_env *env,  in check_ptr_to_map_access()  argument
6752 		verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");  in check_ptr_to_map_access()
6757 		verbose(env, "map_ptr access not supported for map type %d\n",  in check_ptr_to_map_access()
6765 	if (!env->allow_ptr_leaks) {  in check_ptr_to_map_access()
6766 		verbose(env,  in check_ptr_to_map_access()
6773 		verbose(env, "R%d is %s invalid negative access: off=%d\n",  in check_ptr_to_map_access()
6779 		verbose(env, "only read from %s is supported\n", tname);  in check_ptr_to_map_access()
6785 	mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0);  in check_ptr_to_map_access()
6786 	ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);  in check_ptr_to_map_access()
6791 		mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);  in check_ptr_to_map_access()
6802 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,  in check_stack_slot_within_bounds()  argument
6809 	if (t == BPF_WRITE || env->allow_uninit_stack)  in check_stack_slot_within_bounds()
6825 		struct bpf_verifier_env *env,  in check_stack_access_within_bounds()  argument
6829 	struct bpf_reg_state *regs = cur_regs(env);  in check_stack_access_within_bounds()
6831 	struct bpf_func_state *state = func(env, reg);  in check_stack_access_within_bounds()
6850 			verbose(env, "invalid unbounded variable-offset%s stack R%d\n",  in check_stack_access_within_bounds()
6858 	err = check_stack_slot_within_bounds(env, min_off, state, type);  in check_stack_access_within_bounds()
6869 			verbose(env, "invalid%s stack R%d off=%d size=%d\n",  in check_stack_access_within_bounds()
6875 			verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n",  in check_stack_access_within_bounds()
6884 	return grow_stack_state(env, state, -min_off /* size */);  in check_stack_access_within_bounds()
6904 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,  in check_mem_access()  argument
6908 	struct bpf_reg_state *regs = cur_regs(env);  in check_mem_access()
6917 	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);  in check_mem_access()
6926 			verbose(env, "write to change key R%d not allowed\n", regno);  in check_mem_access()
6930 		err = check_mem_region_access(env, regno, off, size,  in check_mem_access()
6935 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
6940 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
6941 			verbose(env, "R%d leaks addr into map\n", value_regno);  in check_mem_access()
6944 		err = check_map_access_type(env, regno, off, size, t);  in check_mem_access()
6947 		err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);  in check_mem_access()
6954 			err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);  in check_mem_access()
6973 				mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
6980 			verbose(env, "R%d invalid mem access '%s'\n", regno,  in check_mem_access()
6981 				reg_type_str(env, reg->type));  in check_mem_access()
6986 			verbose(env, "R%d cannot write into %s\n",  in check_mem_access()
6987 				regno, reg_type_str(env, reg->type));  in check_mem_access()
6992 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
6993 			verbose(env, "R%d leaks addr into mem\n", value_regno);  in check_mem_access()
6997 		err = check_mem_region_access(env, regno, off, size,  in check_mem_access()
7000 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7009 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
7010 			verbose(env, "R%d leaks addr into ctx\n", value_regno);  in check_mem_access()
7014 		err = check_ptr_off_reg(env, reg, regno);  in check_mem_access()
7018 		err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf,  in check_mem_access()
7021 			verbose_linfo(env, insn_idx, "; ");  in check_mem_access()
7028 				if (is_retval && get_func_retval_range(env->prog, &range)) {  in check_mem_access()
7029 					err = __mark_reg_s32_range(env, regs, value_regno,  in check_mem_access()
7034 					mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7037 				mark_reg_known_zero(env, regs,  in check_mem_access()
7040 					regs[value_regno].id = ++env->id_gen;  in check_mem_access()
7057 		err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);  in check_mem_access()
7062 			err = check_stack_read(env, regno, off, size,  in check_mem_access()
7065 			err = check_stack_write(env, regno, off, size,  in check_mem_access()
7068 		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {  in check_mem_access()
7069 			verbose(env, "cannot write into packet\n");  in check_mem_access()
7073 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
7074 			verbose(env, "R%d leaks addr into packet\n",  in check_mem_access()
7078 		err = check_packet_access(env, regno, off, size, false);  in check_mem_access()
7080 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7083 		    is_pointer_value(env, value_regno)) {  in check_mem_access()
7084 			verbose(env, "R%d leaks addr into flow keys\n",  in check_mem_access()
7089 		err = check_flow_keys_access(env, off, size);  in check_mem_access()
7091 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7094 			verbose(env, "R%d cannot write into %s\n",  in check_mem_access()
7095 				regno, reg_type_str(env, reg->type));  in check_mem_access()
7098 		err = check_sock_access(env, insn_idx, regno, off, size, t);  in check_mem_access()
7100 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7102 		err = check_tp_buffer_access(env, reg, regno, off, size);  in check_mem_access()
7104 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7107 		err = check_ptr_to_btf_access(env, regs, regno, off, size, t,  in check_mem_access()
7110 		err = check_ptr_to_map_access(env, regs, regno, off, size, t,  in check_mem_access()
7118 				verbose(env, "R%d cannot write into %s\n",  in check_mem_access()
7119 					regno, reg_type_str(env, reg->type));  in check_mem_access()
7122 			max_access = &env->prog->aux->max_rdonly_access;  in check_mem_access()
7124 			max_access = &env->prog->aux->max_rdwr_access;  in check_mem_access()
7127 		err = check_buffer_access(env, reg, regno, off, size, false,  in check_mem_access()
7131 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7134 			mark_reg_unknown(env, regs, value_regno);  in check_mem_access()
7136 		verbose(env, "R%d invalid mem access '%s'\n", regno,  in check_mem_access()
7137 			reg_type_str(env, reg->type));  in check_mem_access()
7152 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
7155 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)  in check_atomic()  argument
7173 		verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);  in check_atomic()
7178 		verbose(env, "invalid atomic operand size\n");  in check_atomic()
7183 	err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_atomic()
7188 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_atomic()
7196 		err = check_reg_arg(env, aux_reg, SRC_OP);  in check_atomic()
7200 		if (is_pointer_value(env, aux_reg)) {  in check_atomic()
7201 			verbose(env, "R%d leaks addr into mem\n", aux_reg);  in check_atomic()
7206 	if (is_pointer_value(env, insn->src_reg)) {  in check_atomic()
7207 		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);  in check_atomic()
7211 	if (is_ctx_reg(env, insn->dst_reg) ||  in check_atomic()
7212 	    is_pkt_reg(env, insn->dst_reg) ||  in check_atomic()
7213 	    is_flow_key_reg(env, insn->dst_reg) ||  in check_atomic()
7214 	    is_sk_reg(env, insn->dst_reg) ||  in check_atomic()
7215 	    (is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) {  in check_atomic()
7216 		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",  in check_atomic()
7218 			reg_type_str(env, reg_state(env, insn->dst_reg)->type));  in check_atomic()
7229 		err = check_reg_arg(env, load_reg, DST_OP);  in check_atomic()
7242 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,  in check_atomic()
7245 		err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,  in check_atomic()
7251 	if (is_arena_reg(env, insn->dst_reg)) {  in check_atomic()
7252 		err = save_aux_ptr_type(env, PTR_TO_ARENA, false);  in check_atomic()
7257 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,  in check_atomic()
7275 		struct bpf_verifier_env *env, int regno, int off,  in check_stack_range_initialized()  argument
7279 	struct bpf_reg_state *reg = reg_state(env, regno);  in check_stack_range_initialized()
7280 	struct bpf_func_state *state = func(env, reg);  in check_stack_range_initialized()
7290 		verbose(env, "invalid zero-sized read\n");  in check_stack_range_initialized()
7304 	err = check_stack_access_within_bounds(env, regno, off, access_size,  in check_stack_range_initialized()
7318 		if (!env->bypass_spec_v1) {  in check_stack_range_initialized()
7322 			verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",  in check_stack_range_initialized()
7361 				verbose(env, "potential write to dynptr at off=%d disallowed\n", i);  in check_stack_range_initialized()
7376 			verbose(env, "verifier bug: allocated_stack too small");  in check_stack_range_initialized()
7384 		    (*stype == STACK_INVALID && env->allow_uninit_stack)) {  in check_stack_range_initialized()
7394 		     env->allow_ptr_leaks)) {  in check_stack_range_initialized()
7396 				__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);  in check_stack_range_initialized()
7404 			verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",  in check_stack_range_initialized()
7410 			verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",  in check_stack_range_initialized()
7418 		mark_reg_read(env, &state->stack[spi].spilled_ptr,  in check_stack_range_initialized()
7430 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,  in check_helper_mem_access()  argument
7435 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_helper_mem_access()
7441 		return check_packet_access(env, regno, reg->off, access_size,  in check_helper_mem_access()
7445 			verbose(env, "R%d cannot write into %s\n", regno,  in check_helper_mem_access()
7446 				reg_type_str(env, reg->type));  in check_helper_mem_access()
7449 		return check_mem_region_access(env, regno, reg->off, access_size,  in check_helper_mem_access()
7452 		if (check_map_access_type(env, regno, reg->off, access_size, access_type))  in check_helper_mem_access()
7454 		return check_map_access(env, regno, reg->off, access_size,  in check_helper_mem_access()
7459 				verbose(env, "R%d cannot write into %s\n", regno,  in check_helper_mem_access()
7460 					reg_type_str(env, reg->type));  in check_helper_mem_access()
7464 		return check_mem_region_access(env, regno, reg->off,  in check_helper_mem_access()
7470 				verbose(env, "R%d cannot write into %s\n", regno,  in check_helper_mem_access()
7471 					reg_type_str(env, reg->type));  in check_helper_mem_access()
7475 			max_access = &env->prog->aux->max_rdonly_access;  in check_helper_mem_access()
7477 			max_access = &env->prog->aux->max_rdwr_access;  in check_helper_mem_access()
7479 		return check_buffer_access(env, reg, regno, reg->off,  in check_helper_mem_access()
7484 				env,  in check_helper_mem_access()
7488 		return check_ptr_to_btf_access(env, regs, regno, reg->off,  in check_helper_mem_access()
7496 		if (!env->ops->convert_ctx_access) {  in check_helper_mem_access()
7503 			return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,  in check_helper_mem_access()
7514 		verbose(env, "R%d type=%s ", regno,  in check_helper_mem_access()
7515 			reg_type_str(env, reg->type));  in check_helper_mem_access()
7516 		verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));  in check_helper_mem_access()
7527 static int check_mem_size_reg(struct bpf_verifier_env *env,  in check_mem_size_reg()  argument
7554 		verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",  in check_mem_size_reg()
7560 		verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n",  in check_mem_size_reg()
7566 		verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",  in check_mem_size_reg()
7570 	err = check_helper_mem_access(env, regno - 1, reg->umax_value,  in check_mem_size_reg()
7573 		err = mark_chain_precision(env, regno);  in check_mem_size_reg()
7577 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_mem_reg()  argument
7596 	err = check_helper_mem_access(env, regno, mem_size, BPF_READ, true, NULL);  in check_mem_reg()
7597 	err = err ?: check_helper_mem_access(env, regno, mem_size, BPF_WRITE, true, NULL);  in check_mem_reg()
7605 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,  in check_kfunc_mem_size_reg()  argument
7608 	struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];  in check_kfunc_mem_size_reg()
7623 	err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta);  in check_kfunc_mem_size_reg()
7624 	err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta);  in check_kfunc_mem_size_reg()
7654 static int process_spin_lock(struct bpf_verifier_env *env, int regno,  in process_spin_lock()  argument
7657 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_spin_lock()
7658 	struct bpf_verifier_state *cur = env->cur_state;  in process_spin_lock()
7666 		verbose(env,  in process_spin_lock()
7674 			verbose(env,  in process_spin_lock()
7685 		verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local",  in process_spin_lock()
7690 		verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n",  in process_spin_lock()
7696 			verbose(env,  in process_spin_lock()
7714 			verbose(env, "bpf_spin_unlock without taking a lock\n");  in process_spin_lock()
7719 			verbose(env, "bpf_spin_unlock of different lock\n");  in process_spin_lock()
7723 		invalidate_non_owning_refs(env);  in process_spin_lock()
7731 static int process_timer_func(struct bpf_verifier_env *env, int regno,  in process_timer_func()  argument
7734 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_timer_func()
7740 		verbose(env,  in process_timer_func()
7746 		verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n",  in process_timer_func()
7751 		verbose(env, "map '%s' has no valid bpf_timer\n", map->name);  in process_timer_func()
7755 		verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n",  in process_timer_func()
7760 		verbose(env, "verifier bug. Two map pointers in a timer helper\n");  in process_timer_func()
7768 static int process_wq_func(struct bpf_verifier_env *env, int regno,  in process_wq_func()  argument
7771 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_wq_func()
7776 		verbose(env, "off %lld doesn't point to 'struct bpf_wq' that is at %d\n",  in process_wq_func()
7785 static int process_kptr_func(struct bpf_verifier_env *env, int regno,  in process_kptr_func()  argument
7788 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_kptr_func()
7799 			verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",  in process_kptr_func()
7808 		verbose(env,  in process_kptr_func()
7815 		verbose(env, "R%d has no valid kptr\n", regno);  in process_kptr_func()
7822 		verbose(env, "off=%d doesn't point to kptr\n", kptr_off);  in process_kptr_func()
7826 		verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);  in process_kptr_func()
7858 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,  in process_dynptr_func()  argument
7861 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_dynptr_func()
7865 		verbose(env,  in process_dynptr_func()
7875 		verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n");  in process_dynptr_func()
7897 		if (!is_dynptr_reg_valid_uninit(env, reg)) {  in process_dynptr_func()
7898 			verbose(env, "Dynptr has to be an uninitialized dynptr\n");  in process_dynptr_func()
7904 			err = check_mem_access(env, insn_idx, regno,  in process_dynptr_func()
7910 		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);  in process_dynptr_func()
7914 			verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");  in process_dynptr_func()
7918 		if (!is_dynptr_reg_valid_init(env, reg)) {  in process_dynptr_func()
7919 			verbose(env,  in process_dynptr_func()
7926 		if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) {  in process_dynptr_func()
7927 			verbose(env,  in process_dynptr_func()
7933 		err = mark_dynptr_read(env, reg);  in process_dynptr_func()
7938 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)  in iter_ref_obj_id()  argument
7940 	struct bpf_func_state *state = func(env, reg);  in iter_ref_obj_id()
7978 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,  in process_iter_arg()  argument
7981 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in process_iter_arg()
7993 		verbose(env, "expected valid iter pointer as arg #%d\n", regno);  in process_iter_arg()
8001 		if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {  in process_iter_arg()
8002 			verbose(env, "expected uninitialized iter_%s as arg #%d\n",  in process_iter_arg()
8008 			err = check_mem_access(env, insn_idx, regno,  in process_iter_arg()
8014 		err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);  in process_iter_arg()
8021 		err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots);  in process_iter_arg()
8026 			verbose(env, "expected an initialized iter_%s as arg #%d\n",  in process_iter_arg()
8030 			verbose(env, "expected an RCU CS when using %s\n", meta->func_name);  in process_iter_arg()
8036 		spi = iter_get_spi(env, reg, nr_slots);  in process_iter_arg()
8040 		err = mark_iter_read(env, reg, spi, nr_slots);  in process_iter_arg()
8047 		meta->ref_obj_id = iter_ref_obj_id(env, reg, spi);  in process_iter_arg()
8050 			err = unmark_stack_slots_iter(env, reg, nr_slots);  in process_iter_arg()
8062 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,  in find_prev_entry()  argument
8070 	sl = *explored_state(env, insn_idx);  in find_prev_entry()
8084 static void reset_idmap_scratch(struct bpf_verifier_env *env);
8089 static void maybe_widen_reg(struct bpf_verifier_env *env,  in maybe_widen_reg()  argument
8099 	__mark_reg_unknown(env, rcur);  in maybe_widen_reg()
8102 static int widen_imprecise_scalars(struct bpf_verifier_env *env,  in widen_imprecise_scalars()  argument
8109 	reset_idmap_scratch(env);  in widen_imprecise_scalars()
8115 			maybe_widen_reg(env,  in widen_imprecise_scalars()
8118 					&env->idmap_scratch);  in widen_imprecise_scalars()
8125 			maybe_widen_reg(env,  in widen_imprecise_scalars()
8128 					&env->idmap_scratch);  in widen_imprecise_scalars()
8221 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,  in process_iter_next_call()  argument
8224 	struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;  in process_iter_next_call()
8234 		verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n",  in process_iter_next_call()
8245 			verbose(env, "bug: bad parent state for iter next call");  in process_iter_next_call()
8252 		prev_st = find_prev_entry(env, cur_st->parent, insn_idx);  in process_iter_next_call()
8254 		queued_st = push_stack(env, insn_idx + 1, insn_idx, false);  in process_iter_next_call()
8262 			widen_imprecise_scalars(env, prev_st, queued_st);  in process_iter_next_call()
8271 	__mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);  in process_iter_next_call()
8298 static int resolve_map_arg_type(struct bpf_verifier_env *env,  in resolve_map_arg_type()  argument
8304 		verbose(env, "invalid map_ptr to access map->type\n");  in resolve_map_arg_type()
8314 			verbose(env, "invalid arg_type for sockmap/sockhash\n");  in resolve_map_arg_type()
8439 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,  in check_reg_type()  argument
8444 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_reg_type()
8451 		verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);  in check_reg_type()
8489 	verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));  in check_reg_type()
8491 		verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));  in check_reg_type()
8492 	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));  in check_reg_type()
8501 			verbose(env,  in check_reg_type()
8504 				regno, reg_type_str(env, reg->type));  in check_reg_type()
8527 			verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno);  in check_reg_type()
8533 				verbose(env, "verifier internal error: missing arg compatible BTF ID\n");  in check_reg_type()
8540 			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))  in check_reg_type()
8544 				verbose(env, "verifier internal error:");  in check_reg_type()
8545 				verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",  in check_reg_type()
8550 			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,  in check_reg_type()
8553 				verbose(env, "R%d is of type %s but %s is expected\n",  in check_reg_type()
8565 			verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n");  in check_reg_type()
8570 			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))  in check_reg_type()
8580 		verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n");  in check_reg_type()
8603 static int check_func_arg_reg_off(struct bpf_verifier_env *env,  in check_func_arg_reg_off()  argument
8630 			verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",  in check_func_arg_reg_off()
8634 		return __check_ptr_off_reg(env, reg, regno, false);  in check_func_arg_reg_off()
8668 		return __check_ptr_off_reg(env, reg, regno, true);  in check_func_arg_reg_off()
8670 		return __check_ptr_off_reg(env, reg, regno, false);  in check_func_arg_reg_off()
8674 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,  in get_dynptr_arg_reg()  argument
8684 				verbose(env, "verifier internal error: multiple dynptr args\n");  in get_dynptr_arg_reg()
8691 		verbose(env, "verifier internal error: no dynptr arg found\n");  in get_dynptr_arg_reg()
8696 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in dynptr_id()  argument
8698 	struct bpf_func_state *state = func(env, reg);  in dynptr_id()
8703 	spi = dynptr_get_spi(env, reg);  in dynptr_id()
8709 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in dynptr_ref_obj_id()  argument
8711 	struct bpf_func_state *state = func(env, reg);  in dynptr_ref_obj_id()
8716 	spi = dynptr_get_spi(env, reg);  in dynptr_ref_obj_id()
8722 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,  in dynptr_get_type()  argument
8725 	struct bpf_func_state *state = func(env, reg);  in dynptr_get_type()
8733 		verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");  in dynptr_get_type()
8740 static int check_reg_const_str(struct bpf_verifier_env *env,  in check_reg_const_str()  argument
8753 		verbose(env, "R%d does not point to a readonly map'\n", regno);  in check_reg_const_str()
8758 		verbose(env, "R%d is not a constant address'\n", regno);  in check_reg_const_str()
8763 		verbose(env, "no direct value access support for this map type\n");  in check_reg_const_str()
8767 	err = check_map_access(env, regno, reg->off,  in check_reg_const_str()
8776 		verbose(env, "direct value access on string failed\n");  in check_reg_const_str()
8782 		verbose(env, "string is not zero-terminated\n");  in check_reg_const_str()
8788 static int check_func_arg(struct bpf_verifier_env *env, u32 arg,  in check_func_arg()  argument
8794 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];  in check_func_arg()
8803 	err = check_reg_arg(env, regno, SRC_OP);  in check_func_arg()
8808 		if (is_pointer_value(env, regno)) {  in check_func_arg()
8809 			verbose(env, "R%d leaks addr into helper function\n",  in check_func_arg()
8817 	    !may_access_direct_pkt_data(env, meta, BPF_READ)) {  in check_func_arg()
8818 		verbose(env, "helper access to the packet is not allowed\n");  in check_func_arg()
8823 		err = resolve_map_arg_type(env, meta, &arg_type);  in check_func_arg()
8839 	err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);  in check_func_arg()
8843 	err = check_func_arg_reg_off(env, reg, regno, arg_type);  in check_func_arg()
8850 			struct bpf_func_state *state = func(env, reg);  in check_func_arg()
8859 				spi = dynptr_get_spi(env, reg);  in check_func_arg()
8861 					verbose(env, "arg %d is an unacquired reference\n", regno);  in check_func_arg()
8865 				verbose(env, "cannot release unowned const bpf_dynptr\n");  in check_func_arg()
8869 			verbose(env, "R%d must be referenced when passed to release function\n",  in check_func_arg()
8874 			verbose(env, "verifier internal error: more than one release argument\n");  in check_func_arg()
8882 			verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",  in check_func_arg()
8908 				verbose(env,  in check_func_arg()
8928 			verbose(env, "invalid map_ptr to access map->key\n");  in check_func_arg()
8931 		err = check_helper_mem_access(env, regno, meta->map_ptr->key_size,  in check_func_arg()
8943 			verbose(env, "invalid map_ptr to access map->value\n");  in check_func_arg()
8947 		err = check_helper_mem_access(env, regno, meta->map_ptr->value_size,  in check_func_arg()
8953 			verbose(env, "Helper has invalid btf_id in R%d\n", regno);  in check_func_arg()
8960 		if (in_rbtree_lock_required_cb(env)) {  in check_func_arg()
8961 			verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");  in check_func_arg()
8965 			err = process_spin_lock(env, regno, true);  in check_func_arg()
8969 			err = process_spin_lock(env, regno, false);  in check_func_arg()
8973 			verbose(env, "verifier internal error\n");  in check_func_arg()
8978 		err = process_timer_func(env, regno, meta);  in check_func_arg()
8991 			err = check_helper_mem_access(env, regno, fn->arg_size[arg],  in check_func_arg()
8997 				err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true);  in check_func_arg()
9001 		err = check_mem_size_reg(env, reg, regno,  in check_func_arg()
9007 		err = check_mem_size_reg(env, reg, regno,  in check_func_arg()
9013 		err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);  in check_func_arg()
9019 			verbose(env, "R%d is not a known constant'\n",  in check_func_arg()
9024 		err = mark_chain_precision(env, regno);  in check_func_arg()
9030 		err = check_reg_const_str(env, reg, regno);  in check_func_arg()
9036 		err = process_kptr_func(env, regno, meta);  in check_func_arg()
9045 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)  in may_update_sockmap()  argument
9047 	enum bpf_attach_type eatype = env->prog->expected_attach_type;  in may_update_sockmap()
9048 	enum bpf_prog_type type = resolve_prog_type(env->prog);  in may_update_sockmap()
9079 	verbose(env, "cannot update sockmap in this context\n");  in may_update_sockmap()
9083 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)  in allow_tail_call_in_subprogs()  argument
9085 	return env->prog->jit_requested &&  in allow_tail_call_in_subprogs()
9089 static int check_map_func_compatibility(struct bpf_verifier_env *env,  in check_map_func_compatibility()  argument
9165 		    !may_update_sockmap(env, func_id))  in check_map_func_compatibility()
9174 		    !may_update_sockmap(env, func_id))  in check_map_func_compatibility()
9226 		if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {  in check_map_func_compatibility()
9227 			verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");  in check_map_func_compatibility()
9335 	verbose(env, "cannot pass map_type %d into func %s#%d\n",  in check_map_func_compatibility()
9427 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)  in clear_all_pkt_pointers()  argument
9432 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({  in clear_all_pkt_pointers()
9434 			mark_reg_invalid(env, reg);  in clear_all_pkt_pointers()
9467 static int release_reference(struct bpf_verifier_env *env,  in release_reference()  argument
9474 	err = release_reference_state(cur_func(env), ref_obj_id);  in release_reference()
9478 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({  in release_reference()
9480 			mark_reg_invalid(env, reg);  in release_reference()
9486 static void invalidate_non_owning_refs(struct bpf_verifier_env *env)  in invalidate_non_owning_refs()  argument
9491 	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({  in invalidate_non_owning_refs()
9493 			mark_reg_invalid(env, reg);  in invalidate_non_owning_refs()
9497 static void clear_caller_saved_regs(struct bpf_verifier_env *env,  in clear_caller_saved_regs()  argument
9504 		mark_reg_not_init(env, regs, caller_saved[i]);  in clear_caller_saved_regs()
9505 		__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);  in clear_caller_saved_regs()
9509 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
9514 static int set_callee_state(struct bpf_verifier_env *env,
9518 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,  in setup_func_entry()  argument
9526 		verbose(env, "the call stack of %d frames is too deep\n",  in setup_func_entry()
9532 		verbose(env, "verifier bug. Frame %d already allocated\n",  in setup_func_entry()
9547 	init_func_state(env, callee,  in setup_func_entry()
9554 	err = err ?: set_callee_state_cb(env, caller, callee, callsite);  in setup_func_entry()
9569 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,  in btf_check_func_arg_match()  argument
9573 	struct bpf_subprog_info *sub = subprog_info(env, subprog);  in btf_check_func_arg_match()
9574 	struct bpf_verifier_log *log = &env->log;  in btf_check_func_arg_match()
9578 	ret = btf_prepare_func_args(env, subprog);  in btf_check_func_arg_match()
9596 			ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);  in btf_check_func_arg_match()
9607 			ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);  in btf_check_func_arg_match()
9610 			if (check_mem_reg(env, reg, regno, arg->mem_size))  in btf_check_func_arg_match()
9629 			ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR);  in btf_check_func_arg_match()
9633 			ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0);  in btf_check_func_arg_match()
9644 			err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);  in btf_check_func_arg_match()
9645 			err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);  in btf_check_func_arg_match()
9665 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,  in btf_check_subprog_call()  argument
9668 	struct bpf_prog *prog = env->prog;  in btf_check_subprog_call()
9683 	err = btf_check_func_arg_match(env, subprog, btf, regs);  in btf_check_subprog_call()
9693 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in push_callback_call()  argument
9697 	struct bpf_verifier_state *state = env->cur_state, *callback_state;  in push_callback_call()
9702 	err = btf_check_subprog_call(env, subprog, caller->regs);  in push_callback_call()
9710 	env->subprog_info[subprog].is_cb = true;  in push_callback_call()
9713 		verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n",  in push_callback_call()
9718 		verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n",  in push_callback_call()
9727 		env->subprog_info[subprog].is_async_cb = true;  in push_callback_call()
9728 		async_cb = push_async_cb(env, env->subprog_info[subprog].start,  in push_callback_call()
9737 		err = set_callee_state_cb(env, caller, callee, insn_idx);  in push_callback_call()
9747 	callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);  in push_callback_call()
9751 	err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,  in push_callback_call()
9762 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_func_call()  argument
9765 	struct bpf_verifier_state *state = env->cur_state;  in check_func_call()
9770 	subprog = find_subprog(env, target_insn);  in check_func_call()
9772 		verbose(env, "verifier bug. No program starts at insn %d\n", target_insn);  in check_func_call()
9777 	err = btf_check_subprog_call(env, subprog, caller->regs);  in check_func_call()
9780 	if (subprog_is_global(env, subprog)) {  in check_func_call()
9781 		const char *sub_name = subprog_name(env, subprog);  in check_func_call()
9784 		if (env->cur_state->active_lock.ptr) {  in check_func_call()
9785 			verbose(env, "global function calls are not allowed while holding a lock,\n"  in check_func_call()
9791 		if (env->cur_state->active_preempt_lock) {  in check_func_call()
9792 			verbose(env, "global function calls are not allowed with preemption disabled,\n"  in check_func_call()
9798 			verbose(env, "Caller passes invalid args into func#%d ('%s')\n",  in check_func_call()
9803 		verbose(env, "Func#%d ('%s') is global and assumed valid.\n",  in check_func_call()
9806 		subprog_aux(env, subprog)->called = true;  in check_func_call()
9807 		clear_caller_saved_regs(env, caller->regs);  in check_func_call()
9810 		mark_reg_unknown(env, caller->regs, BPF_REG_0);  in check_func_call()
9820 	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);  in check_func_call()
9824 	clear_caller_saved_regs(env, caller->regs);  in check_func_call()
9827 	*insn_idx = env->subprog_info[subprog].start - 1;  in check_func_call()
9829 	if (env->log.level & BPF_LOG_LEVEL) {  in check_func_call()
9830 		verbose(env, "caller:\n");  in check_func_call()
9831 		print_verifier_state(env, caller, true);  in check_func_call()
9832 		verbose(env, "callee:\n");  in check_func_call()
9833 		print_verifier_state(env, state->frame[state->curframe], true);  in check_func_call()
9839 int map_set_for_each_callback_args(struct bpf_verifier_env *env,  in map_set_for_each_callback_args()  argument
9862 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in map_set_for_each_callback_args()
9866 static int set_callee_state(struct bpf_verifier_env *env,  in set_callee_state()  argument
9880 static int set_map_elem_callback_state(struct bpf_verifier_env *env,  in set_map_elem_callback_state()  argument
9885 	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];  in set_map_elem_callback_state()
9893 		verbose(env, "callback function not allowed for map\n");  in set_map_elem_callback_state()
9897 	err = map->ops->map_set_for_each_callback_args(env, caller, callee);  in set_map_elem_callback_state()
9906 static int set_loop_callback_state(struct bpf_verifier_env *env,  in set_loop_callback_state()  argument
9919 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);  in set_loop_callback_state()
9920 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);  in set_loop_callback_state()
9921 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in set_loop_callback_state()
9928 static int set_timer_callback_state(struct bpf_verifier_env *env,  in set_timer_callback_state()  argument
9951 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);  in set_timer_callback_state()
9952 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in set_timer_callback_state()
9958 static int set_find_vma_callback_state(struct bpf_verifier_env *env,  in set_find_vma_callback_state()  argument
9979 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);  in set_find_vma_callback_state()
9980 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in set_find_vma_callback_state()
9986 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,  in set_user_ringbuf_callback_state()  argument
9995 	__mark_reg_not_init(env, &callee->regs[BPF_REG_0]);  in set_user_ringbuf_callback_state()
9996 	mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);  in set_user_ringbuf_callback_state()
10000 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);  in set_user_ringbuf_callback_state()
10001 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);  in set_user_ringbuf_callback_state()
10002 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in set_user_ringbuf_callback_state()
10009 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,  in set_rbtree_add_callback_state()  argument
10029 	ref_set_non_owning(env, &callee->regs[BPF_REG_1]);  in set_rbtree_add_callback_state()
10031 	ref_set_non_owning(env, &callee->regs[BPF_REG_2]);  in set_rbtree_add_callback_state()
10033 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);  in set_rbtree_add_callback_state()
10034 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);  in set_rbtree_add_callback_state()
10035 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);  in set_rbtree_add_callback_state()
10047 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)  in in_rbtree_lock_required_cb()  argument
10049 	struct bpf_verifier_state *state = env->cur_state;  in in_rbtree_lock_required_cb()
10050 	struct bpf_insn *insn = env->prog->insnsi;  in in_rbtree_lock_required_cb()
10075 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)  in prepare_func_exit()  argument
10077 	struct bpf_verifier_state *state = env->cur_state, *prev_st;  in prepare_func_exit()
10092 		verbose(env, "cannot return stack pointer to the caller\n");  in prepare_func_exit()
10099 			verbose(env, "R0 not a scalar value\n");  in prepare_func_exit()
10104 		err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64);  in prepare_func_exit()
10105 		err = err ?: mark_chain_precision(env, BPF_REG_0);  in prepare_func_exit()
10111 			verbose_invalid_scalar(env, r0, callee->callback_ret_range,  in prepare_func_exit()
10115 		if (!calls_callback(env, callee->callsite)) {  in prepare_func_exit()
10116 			verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n",  in prepare_func_exit()
10147 	if (env->log.level & BPF_LOG_LEVEL) {  in prepare_func_exit()
10148 		verbose(env, "returning from callee:\n");  in prepare_func_exit()
10149 		print_verifier_state(env, callee, true);  in prepare_func_exit()
10150 		verbose(env, "to caller at %d:\n", *insn_idx);  in prepare_func_exit()
10151 		print_verifier_state(env, caller, true);  in prepare_func_exit()
10169 	prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;  in prepare_func_exit()
10171 		err = widen_imprecise_scalars(env, prev_st, state);  in prepare_func_exit()
10178 static int do_refine_retval_range(struct bpf_verifier_env *env,  in do_refine_retval_range()  argument
10213 	return reg_bounds_sanity_check(env, ret_reg, "retval");  in do_refine_retval_range()
10217 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,  in record_func_map()  argument
10220 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];  in record_func_map()
10236 		verbose(env, "kernel subsystem misconfigured verifier\n");  in record_func_map()
10249 		verbose(env, "write into map forbidden\n");  in record_func_map()
10263 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,  in record_func_key()  argument
10266 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];  in record_func_key()
10267 	struct bpf_reg_state *regs = cur_regs(env), *reg;  in record_func_key()
10275 		verbose(env, "kernel subsystem misconfigured verifier\n");  in record_func_key()
10288 	err = mark_chain_precision(env, BPF_REG_3);  in record_func_key()
10299 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)  in check_reference_leak()  argument
10301 	struct bpf_func_state *state = cur_func(env);  in check_reference_leak()
10311 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",  in check_reference_leak()
10318 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,  in check_bpf_snprintf_call()  argument
10341 		verbose(env, "verifier bug\n");  in check_bpf_snprintf_call()
10351 		verbose(env, "Invalid format string\n");  in check_bpf_snprintf_call()
10356 static int check_get_func_ip(struct bpf_verifier_env *env)  in check_get_func_ip()  argument
10358 	enum bpf_prog_type type = resolve_prog_type(env->prog);  in check_get_func_ip()
10362 		if (!bpf_prog_has_trampoline(env->prog)) {  in check_get_func_ip()
10363 			verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",  in check_get_func_ip()
10372 	verbose(env, "func %s#%d not supported for program type %d\n",  in check_get_func_ip()
10377 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)  in cur_aux()  argument
10379 	return &env->insn_aux_data[env->insn_idx];  in cur_aux()
10382 static bool loop_flag_is_zero(struct bpf_verifier_env *env)  in loop_flag_is_zero()  argument
10384 	struct bpf_reg_state *regs = cur_regs(env);  in loop_flag_is_zero()
10389 		mark_chain_precision(env, BPF_REG_4);  in loop_flag_is_zero()
10394 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)  in update_loop_inline_state()  argument
10396 	struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;  in update_loop_inline_state()
10400 		state->fit_for_inline = loop_flag_is_zero(env);  in update_loop_inline_state()
10408 	state->fit_for_inline = (loop_flag_is_zero(env) &&  in update_loop_inline_state()
10412 static int get_helper_proto(struct bpf_verifier_env *env, int func_id,  in get_helper_proto()  argument
10418 	if (!env->ops->get_func_proto)  in get_helper_proto()
10421 	*ptr = env->ops->get_func_proto(func_id, env->prog);  in get_helper_proto()
10425 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_helper_call()  argument
10428 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);  in check_helper_call()
10441 	err = get_helper_proto(env, insn->imm, &fn);  in check_helper_call()
10443 		verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id);  in check_helper_call()
10448 		verbose(env, "program of this type cannot use helper %s#%d\n",  in check_helper_call()
10454 	if (!env->prog->gpl_compatible && fn->gpl_only) {  in check_helper_call()
10455 		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");  in check_helper_call()
10459 	if (fn->allowed && !fn->allowed(env->prog)) {  in check_helper_call()
10460 		verbose(env, "helper call is not allowed in probe\n");  in check_helper_call()
10464 	if (!in_sleepable(env) && fn->might_sleep) {  in check_helper_call()
10465 		verbose(env, "helper call might sleep in a non-sleepable prog\n");  in check_helper_call()
10472 		verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",  in check_helper_call()
10482 		verbose(env, "kernel subsystem misconfigured func %s#%d\n",  in check_helper_call()
10487 	if (env->cur_state->active_rcu_lock) {  in check_helper_call()
10489 			verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",  in check_helper_call()
10494 		if (in_sleepable(env) && is_storage_get_function(func_id))  in check_helper_call()
10495 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;  in check_helper_call()
10498 	if (env->cur_state->active_preempt_lock) {  in check_helper_call()
10500 			verbose(env, "sleepable helper %s#%d in non-preemptible region\n",  in check_helper_call()
10505 		if (in_sleepable(env) && is_storage_get_function(func_id))  in check_helper_call()
10506 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;  in check_helper_call()
10512 		err = check_func_arg(env, i, &meta, fn, insn_idx);  in check_helper_call()
10517 	err = record_func_map(env, &meta, func_id, insn_idx);  in check_helper_call()
10521 	err = record_func_key(env, &meta, func_id, insn_idx);  in check_helper_call()
10529 		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,  in check_helper_call()
10535 	regs = cur_regs(env);  in check_helper_call()
10545 				verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n");  in check_helper_call()
10548 			err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);  in check_helper_call()
10551 			bool in_rcu = in_rcu_cs(env);  in check_helper_call()
10555 			err = release_reference_state(cur_func(env), ref_obj_id);  in check_helper_call()
10557 				bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({  in check_helper_call()
10564 							mark_reg_invalid(env, reg);  in check_helper_call()
10570 			err = release_reference(env, meta.ref_obj_id);  in check_helper_call()
10578 			verbose(env, "func %s#%d reference has not been acquired before\n",  in check_helper_call()
10586 		err = check_reference_leak(env, false);  in check_helper_call()
10588 			verbose(env, "tail_call would lead to reference leak\n");  in check_helper_call()
10597 			verbose(env, "get_local_storage() doesn't support non-zero flags\n");  in check_helper_call()
10602 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
10606 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
10610 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
10614 		err = check_bpf_snprintf_call(env, regs);  in check_helper_call()
10617 		update_loop_inline_state(env, meta.subprogno);  in check_helper_call()
10621 		err = mark_chain_precision(env, BPF_REG_1);  in check_helper_call()
10624 		if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {  in check_helper_call()
10625 			err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
10628 			cur_func(env)->callback_depth = 0;  in check_helper_call()
10629 			if (env->log.level & BPF_LOG_LEVEL2)  in check_helper_call()
10630 				verbose(env, "frame%d bpf_loop iteration limit reached\n",  in check_helper_call()
10631 					env->cur_state->curframe);  in check_helper_call()
10636 			verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",  in check_helper_call()
10637 				reg_type_str(env, regs[BPF_REG_1].type));  in check_helper_call()
10643 		    env->prog->expected_attach_type == BPF_LSM_CGROUP) {  in check_helper_call()
10644 			if (!env->prog->aux->attach_func_proto->type) {  in check_helper_call()
10648 				verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");  in check_helper_call()
10658 		reg = get_dynptr_arg_reg(env, fn, regs);  in check_helper_call()
10664 			verbose(env, "verifier internal error: meta.dynptr_id already set\n");  in check_helper_call()
10668 			verbose(env, "verifier internal error: meta.ref_obj_id already set\n");  in check_helper_call()
10672 		id = dynptr_id(env, reg);  in check_helper_call()
10674 			verbose(env, "verifier internal error: failed to obtain dynptr id\n");  in check_helper_call()
10678 		ref_obj_id = dynptr_ref_obj_id(env, reg);  in check_helper_call()
10680 			verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");  in check_helper_call()
10694 		reg = get_dynptr_arg_reg(env, fn, regs);  in check_helper_call()
10698 		dynptr_type = dynptr_get_type(env, reg);  in check_helper_call()
10719 				verbose(env, "Helper has invalid btf/btf_id in R1\n");  in check_helper_call()
10723 			env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;  in check_helper_call()
10728 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
10738 		mark_reg_not_init(env, regs, caller_saved[i]);  in check_helper_call()
10739 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);  in check_helper_call()
10752 		mark_reg_unknown(env, regs, BPF_REG_0);  in check_helper_call()
10759 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
10765 			verbose(env,  in check_helper_call()
10774 			regs[BPF_REG_0].id = ++env->id_gen;  in check_helper_call()
10778 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
10782 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
10786 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
10790 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
10798 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
10809 				verbose(env, "unable to resolve the size of type '%s': %ld\n",  in check_helper_call()
10838 		mark_reg_known_zero(env, regs, BPF_REG_0);  in check_helper_call()
10850 				verbose(env, "verifier internal error:");  in check_helper_call()
10851 				verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n",  in check_helper_call()
10859 			verbose(env, "invalid return type %u of func %s#%d\n",  in check_helper_call()
10869 		verbose(env, "unknown return type %u of func %s#%d\n",  in check_helper_call()
10875 		regs[BPF_REG_0].id = ++env->id_gen;  in check_helper_call()
10878 		verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n",  in check_helper_call()
10890 		int id = acquire_reference_state(env, insn_idx);  in check_helper_call()
10900 	err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);  in check_helper_call()
10904 	err = check_map_func_compatibility(env, meta.map_ptr, func_id);  in check_helper_call()
10910 	    !env->prog->has_callchain_buf) {  in check_helper_call()
10921 			verbose(env, err_str, func_id_name(func_id), func_id);  in check_helper_call()
10925 		env->prog->has_callchain_buf = true;  in check_helper_call()
10929 		env->prog->call_get_stack = true;  in check_helper_call()
10932 		if (check_get_func_ip(env))  in check_helper_call()
10934 		env->prog->call_get_func_ip = true;  in check_helper_call()
10938 		clear_all_pkt_pointers(env);  in check_helper_call()
10945 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,  in mark_btf_func_reg_size()  argument
10948 	struct bpf_reg_state *reg = &cur_regs(env)[regno];  in mark_btf_func_reg_size()
10954 			DEF_NOT_SUBREG : env->insn_idx + 1;  in mark_btf_func_reg_size()
10958 			mark_insn_zext(env, reg);  in mark_btf_func_reg_size()
10959 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);  in mark_btf_func_reg_size()
10961 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);  in mark_btf_func_reg_size()
11155 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,  in is_kfunc_arg_callback()  argument
11168 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,  in __btf_type_is_scalar_struct()  argument
11185 				verbose(env, "max struct nesting depth exceeded\n");  in __btf_type_is_scalar_struct()
11188 			if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))  in __btf_type_is_scalar_struct()
11352 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,  in get_kfunc_ptr_arg_type()  argument
11359 	struct bpf_reg_state *regs = cur_regs(env);  in get_kfunc_ptr_arg_type()
11371 	if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno))  in get_kfunc_ptr_arg_type()
11412 			verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",  in get_kfunc_ptr_arg_type()
11419 	if (is_kfunc_arg_callback(env, meta->btf, &args[argno]))  in get_kfunc_ptr_arg_type()
11432 	if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&  in get_kfunc_ptr_arg_type()
11434 		verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",  in get_kfunc_ptr_arg_type()
11441 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,  in process_kf_arg_ptr_to_btf_id()  argument
11489 	    btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))  in process_kf_arg_ptr_to_btf_id()
11498 …struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, st…  in process_kf_arg_ptr_to_btf_id()
11505 …verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\…  in process_kf_arg_ptr_to_btf_id()
11513 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in ref_set_non_owning()  argument
11515 	struct bpf_verifier_state *state = env->cur_state;  in ref_set_non_owning()
11519 		verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");  in ref_set_non_owning()
11524 		verbose(env, "verifier internal error: NON_OWN_REF already set\n");  in ref_set_non_owning()
11535 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)  in ref_convert_owning_non_owning()  argument
11541 	state = cur_func(env);  in ref_convert_owning_non_owning()
11544 		verbose(env, "verifier internal error: ref_obj_id is zero for "  in ref_convert_owning_non_owning()
11556 		bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({  in ref_convert_owning_non_owning()
11559 				ref_set_non_owning(env, reg);  in ref_convert_owning_non_owning()
11565 	verbose(env, "verifier internal error: ref state missing for ref_obj_id\n");  in ref_convert_owning_non_owning()
11613 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)  in check_reg_allocation_locked()  argument
11626 		verbose(env, "verifier internal error: unknown reg type for lock check\n");  in check_reg_allocation_locked()
11631 	if (!env->cur_state->active_lock.ptr)  in check_reg_allocation_locked()
11633 	if (env->cur_state->active_lock.ptr != ptr ||  in check_reg_allocation_locked()
11634 	    env->cur_state->active_lock.id != id) {  in check_reg_allocation_locked()
11635 		verbose(env, "held lock and object are not in the same allocation\n");  in check_reg_allocation_locked()
11694 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,  in check_kfunc_is_graph_root_api()  argument
11708 		verbose(env, "verifier internal error: unexpected graph root argument type %s\n",  in check_kfunc_is_graph_root_api()
11714 		verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",  in check_kfunc_is_graph_root_api()
11719 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,  in check_kfunc_is_graph_node_api()  argument
11735 		verbose(env, "verifier internal error: unexpected graph node argument type %s\n",  in check_kfunc_is_graph_node_api()
11741 		verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",  in check_kfunc_is_graph_node_api()
11747 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,  in __process_kf_arg_ptr_to_graph_root()  argument
11759 		verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");  in __process_kf_arg_ptr_to_graph_root()
11763 	if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))  in __process_kf_arg_ptr_to_graph_root()
11768 		verbose(env,  in __process_kf_arg_ptr_to_graph_root()
11778 		verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);  in __process_kf_arg_ptr_to_graph_root()
11783 	if (check_reg_allocation_locked(env, reg)) {  in __process_kf_arg_ptr_to_graph_root()
11784 		verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",  in __process_kf_arg_ptr_to_graph_root()
11790 		verbose(env, "verifier internal error: repeating %s arg\n", head_type_name);  in __process_kf_arg_ptr_to_graph_root()
11797 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,  in process_kf_arg_ptr_to_list_head()  argument
11801 	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD,  in process_kf_arg_ptr_to_list_head()
11805 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,  in process_kf_arg_ptr_to_rbtree_root()  argument
11809 	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT,  in process_kf_arg_ptr_to_rbtree_root()
11814 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,  in __process_kf_arg_ptr_to_graph_node()  argument
11827 		verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");  in __process_kf_arg_ptr_to_graph_node()
11831 	if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))  in __process_kf_arg_ptr_to_graph_node()
11836 		verbose(env,  in __process_kf_arg_ptr_to_graph_node()
11845 		verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);  in __process_kf_arg_ptr_to_graph_node()
11853 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,  in __process_kf_arg_ptr_to_graph_node()
11855 		verbose(env, "operation on %s expects arg#1 %s at offset=%d "  in __process_kf_arg_ptr_to_graph_node()
11868 		verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",  in __process_kf_arg_ptr_to_graph_node()
11878 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,  in process_kf_arg_ptr_to_list_node()  argument
11882 	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,  in process_kf_arg_ptr_to_list_node()
11887 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,  in process_kf_arg_ptr_to_rbtree_node()  argument
11891 	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,  in process_kf_arg_ptr_to_rbtree_node()
11902 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)  in check_css_task_iter_allowlist()  argument
11904 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);  in check_css_task_iter_allowlist()
11910 		if (env->prog->expected_attach_type == BPF_TRACE_ITER)  in check_css_task_iter_allowlist()
11914 		return in_sleepable(env);  in check_css_task_iter_allowlist()
11918 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,  in check_kfunc_args()  argument
11931 		verbose(env, "Function %s has %d > %d args\n", func_name, nargs,  in check_kfunc_args()
11940 		struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];  in check_kfunc_args()
11954 				verbose(env, "R%d is not a scalar\n", regno);  in check_kfunc_args()
11960 					verbose(env, "verifier internal error: only one constant argument permitted\n");  in check_kfunc_args()
11964 					verbose(env, "R%d must be a known constant\n", regno);  in check_kfunc_args()
11967 				ret = mark_chain_precision(env, regno);  in check_kfunc_args()
11981 					verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");  in check_kfunc_args()
11986 					verbose(env, "R%d is not a const\n", regno);  in check_kfunc_args()
11991 				ret = mark_chain_precision(env, regno);  in check_kfunc_args()
11999 			verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t));  in check_kfunc_args()
12006 			verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);  in check_kfunc_args()
12012 				verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",  in check_kfunc_args()
12025 		kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);  in check_kfunc_args()
12034 				verbose(env, "pointer in R%d isn't map pointer\n", regno);  in check_kfunc_args()
12052 					verbose(env,  in check_kfunc_args()
12068 					verbose(env, "R%d must be referenced or trusted\n", regno);  in check_kfunc_args()
12072 					verbose(env, "R%d must be a rcu pointer\n", regno);  in check_kfunc_args()
12098 		ret = check_func_arg_reg_off(env, reg, regno, arg_type);  in check_kfunc_args()
12105 				verbose(env, "arg#%d expected pointer to ctx, but got %s\n",  in check_kfunc_args()
12106 					i, reg_type_str(env, reg->type));  in check_kfunc_args()
12111 				ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));  in check_kfunc_args()
12120 					verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i);  in check_kfunc_args()
12125 					verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i);  in check_kfunc_args()
12129 				verbose(env, "arg#%d expected pointer to allocated object\n", i);  in check_kfunc_args()
12133 				verbose(env, "allocated object must be referenced\n");  in check_kfunc_args()
12161 					verbose(env, "verifier internal error: no dynptr type for parent of clone\n");  in check_kfunc_args()
12168 					verbose(env, "verifier internal error: missing ref obj id for parent of clone\n");  in check_kfunc_args()
12173 			ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);  in check_kfunc_args()
12178 				int id = dynptr_id(env, reg);  in check_kfunc_args()
12181 					verbose(env, "verifier internal error: failed to obtain dynptr id\n");  in check_kfunc_args()
12185 				meta->initialized_dynptr.type = dynptr_get_type(env, reg);  in check_kfunc_args()
12186 				meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg);  in check_kfunc_args()
12193 				if (!check_css_task_iter_allowlist(env)) {  in check_kfunc_args()
12194 					verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");  in check_kfunc_args()
12198 			ret = process_iter_arg(env, regno, insn_idx, meta);  in check_kfunc_args()
12205 				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);  in check_kfunc_args()
12209 				verbose(env, "allocated object must be referenced\n");  in check_kfunc_args()
12212 			ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta);  in check_kfunc_args()
12219 				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);  in check_kfunc_args()
12223 				verbose(env, "allocated object must be referenced\n");  in check_kfunc_args()
12226 			ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta);  in check_kfunc_args()
12232 				verbose(env, "arg#%d expected pointer to allocated object\n", i);  in check_kfunc_args()
12236 				verbose(env, "allocated object must be referenced\n");  in check_kfunc_args()
12239 			ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta);  in check_kfunc_args()
12246 					verbose(env, "rbtree_remove node input must be non-owning ref\n");  in check_kfunc_args()
12249 				if (in_rbtree_lock_required_cb(env)) {  in check_kfunc_args()
12250 					verbose(env, "rbtree_remove not allowed in rbtree cb\n");  in check_kfunc_args()
12255 					verbose(env, "arg#%d expected pointer to allocated object\n", i);  in check_kfunc_args()
12259 					verbose(env, "allocated object must be referenced\n");  in check_kfunc_args()
12264 			ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta);  in check_kfunc_args()
12279 				verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type));  in check_kfunc_args()
12280 				verbose(env, "expected %s or socket\n",  in check_kfunc_args()
12281 					reg_type_str(env, base_type(reg->type) |  in check_kfunc_args()
12285 			ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);  in check_kfunc_args()
12292 				verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",  in check_kfunc_args()
12296 			ret = check_mem_reg(env, reg, regno, type_size);  in check_kfunc_args()
12308 				ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1);  in check_kfunc_args()
12310 					verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1);  in check_kfunc_args()
12317 					verbose(env, "verifier internal error: only one constant argument permitted\n");  in check_kfunc_args()
12321 					verbose(env, "R%d must be a known constant\n", regno + 1);  in check_kfunc_args()
12334 				verbose(env, "arg%d expected pointer to func\n", i);  in check_kfunc_args()
12341 				verbose(env, "arg#%d is neither owning or non-owning ref\n", i);  in check_kfunc_args()
12349 				verbose(env, "verifier internal error: Couldn't find btf_record\n");  in check_kfunc_args()
12354 				verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i);  in check_kfunc_args()
12363 				verbose(env, "arg#%d doesn't point to a const string\n", i);  in check_kfunc_args()
12366 			ret = check_reg_const_str(env, reg, regno);  in check_kfunc_args()
12372 				verbose(env, "arg#%d doesn't point to a map value\n", i);  in check_kfunc_args()
12375 			ret = process_wq_func(env, regno, meta);  in check_kfunc_args()
12383 		verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n",  in check_kfunc_args()
12391 static int fetch_kfunc_meta(struct bpf_verifier_env *env,  in fetch_kfunc_meta()  argument
12407 	desc_btf = find_kfunc_desc_btf(env, insn->off);  in fetch_kfunc_meta()
12418 	kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog);  in fetch_kfunc_meta()
12433 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
12435 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_kfunc_call()  argument
12440 	struct bpf_reg_state *regs = cur_regs(env);  in check_kfunc_call()
12454 	err = fetch_kfunc_meta(env, insn, &meta, &func_name);  in check_kfunc_call()
12456 		verbose(env, "calling kernel function %s is not allowed\n", func_name);  in check_kfunc_call()
12460 	insn_aux = &env->insn_aux_data[insn_idx];  in check_kfunc_call()
12465 		verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");  in check_kfunc_call()
12470 	if (sleepable && !in_sleepable(env)) {  in check_kfunc_call()
12471 		verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);  in check_kfunc_call()
12476 	err = check_kfunc_args(env, &meta, insn_idx);  in check_kfunc_call()
12481 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_kfunc_call()
12484 			verbose(env, "kfunc %s#%d failed callback verification\n",  in check_kfunc_call()
12496 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_kfunc_call()
12499 			verbose(env, "kfunc %s#%d failed callback verification\n",  in check_kfunc_call()
12511 	if (env->cur_state->active_rcu_lock) {  in check_kfunc_call()
12516 		if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {  in check_kfunc_call()
12517 			verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");  in check_kfunc_call()
12522 			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);  in check_kfunc_call()
12525 			bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({  in check_kfunc_call()
12531 			env->cur_state->active_rcu_lock = false;  in check_kfunc_call()
12533 			verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);  in check_kfunc_call()
12537 		env->cur_state->active_rcu_lock = true;  in check_kfunc_call()
12539 		verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);  in check_kfunc_call()
12543 	if (env->cur_state->active_preempt_lock) {  in check_kfunc_call()
12545 			env->cur_state->active_preempt_lock++;  in check_kfunc_call()
12547 			env->cur_state->active_preempt_lock--;  in check_kfunc_call()
12549 			verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name);  in check_kfunc_call()
12553 		env->cur_state->active_preempt_lock++;  in check_kfunc_call()
12555 		verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name);  in check_kfunc_call()
12563 		err = release_reference(env, regs[meta.release_regno].ref_obj_id);  in check_kfunc_call()
12565 			verbose(env, "kfunc %s#%d reference has not been acquired before\n",  in check_kfunc_call()
12577 		err = ref_convert_owning_non_owning(env, release_ref_obj_id);  in check_kfunc_call()
12579 			verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",  in check_kfunc_call()
12584 		err = release_reference(env, release_ref_obj_id);  in check_kfunc_call()
12586 			verbose(env, "kfunc %s#%d reference has not been acquired before\n",  in check_kfunc_call()
12594 			verbose(env, "JIT does not support calling kfunc %s#%d\n",  in check_kfunc_call()
12598 		env->seen_exception = true;  in check_kfunc_call()
12603 		if (!env->exception_callback_subprog) {  in check_kfunc_call()
12604 			err = check_return_code(env, BPF_REG_1, "R1");  in check_kfunc_call()
12611 		mark_reg_not_init(env, regs, caller_saved[i]);  in check_kfunc_call()
12622 			verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");  in check_kfunc_call()
12628 		mark_reg_unknown(env, regs, BPF_REG_0);  in check_kfunc_call()
12629 		mark_btf_func_reg_size(env, BPF_REG_0, t->size);  in check_kfunc_call()
12644 					verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");  in check_kfunc_call()
12648 				ret_btf = env->prog->aux->btf;  in check_kfunc_call()
12653 					verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");  in check_kfunc_call()
12659 					verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");  in check_kfunc_call()
12665 						verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",  in check_kfunc_call()
12694 					if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {  in check_kfunc_call()
12695 						verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");  in check_kfunc_call()
12700 						verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");  in check_kfunc_call()
12705 				mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
12715 				mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
12734 				mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
12741 					verbose(env,  in check_kfunc_call()
12746 				mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
12754 				mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
12757 					verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n");  in check_kfunc_call()
12769 					/* this will set env->seen_direct_write to true */  in check_kfunc_call()
12770 					if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {  in check_kfunc_call()
12771 						verbose(env, "the prog does not allow writes to packet data\n");  in check_kfunc_call()
12777 					verbose(env, "verifier internal error: no dynptr id\n");  in check_kfunc_call()
12787 				verbose(env, "kernel function %s unhandled dynamic return type\n",  in check_kfunc_call()
12793 			mark_reg_unknown(env, regs, BPF_REG_0);  in check_kfunc_call()
12806 				verbose(env,  in check_kfunc_call()
12814 			mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
12825 			mark_reg_known_zero(env, regs, BPF_REG_0);  in check_kfunc_call()
12833 				cur_iter = get_iter_from_state(env->cur_state, &meta);  in check_kfunc_call()
12845 			regs[BPF_REG_0].id = ++env->id_gen;  in check_kfunc_call()
12847 		mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));  in check_kfunc_call()
12849 			int id = acquire_reference_state(env, insn_idx);  in check_kfunc_call()
12857 			ref_set_non_owning(env, &regs[BPF_REG_0]);  in check_kfunc_call()
12861 			regs[BPF_REG_0].id = ++env->id_gen;  in check_kfunc_call()
12880 			mark_btf_func_reg_size(env, regno, sizeof(void *));  in check_kfunc_call()
12883 			mark_btf_func_reg_size(env, regno, t->size);  in check_kfunc_call()
12887 		err = process_iter_next_call(env, insn_idx, &meta);  in check_kfunc_call()
12895 static bool check_reg_sane_offset(struct bpf_verifier_env *env,  in check_reg_sane_offset()  argument
12904 		verbose(env, "math between %s pointer and %lld is not allowed\n",  in check_reg_sane_offset()
12905 			reg_type_str(env, type), val);  in check_reg_sane_offset()
12910 		verbose(env, "%s pointer offset %d is not allowed\n",  in check_reg_sane_offset()
12911 			reg_type_str(env, type), reg->off);  in check_reg_sane_offset()
12916 		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",  in check_reg_sane_offset()
12917 			reg_type_str(env, type));  in check_reg_sane_offset()
12922 		verbose(env, "value %lld makes %s pointer be out of bounds\n",  in check_reg_sane_offset()
12923 			smin, reg_type_str(env, type));  in check_reg_sane_offset()
12969 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,  in can_skip_alu_sanitation()  argument
12972 	return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;  in can_skip_alu_sanitation()
12992 static int sanitize_val_alu(struct bpf_verifier_env *env,  in sanitize_val_alu()  argument
12995 	struct bpf_insn_aux_data *aux = cur_aux(env);  in sanitize_val_alu()
12997 	if (can_skip_alu_sanitation(env, insn))  in sanitize_val_alu()
13014 sanitize_speculative_path(struct bpf_verifier_env *env,  in sanitize_speculative_path()  argument
13021 	branch = push_stack(env, next_idx, curr_idx, true);  in sanitize_speculative_path()
13025 			mark_reg_unknown(env, regs, insn->dst_reg);  in sanitize_speculative_path()
13027 			mark_reg_unknown(env, regs, insn->dst_reg);  in sanitize_speculative_path()
13028 			mark_reg_unknown(env, regs, insn->src_reg);  in sanitize_speculative_path()
13034 static int sanitize_ptr_alu(struct bpf_verifier_env *env,  in sanitize_ptr_alu()  argument
13042 	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;  in sanitize_ptr_alu()
13043 	struct bpf_verifier_state *vstate = env->cur_state;  in sanitize_ptr_alu()
13053 	if (can_skip_alu_sanitation(env, insn))  in sanitize_ptr_alu()
13092 			env->explore_alu_limits = true;  in sanitize_ptr_alu()
13123 	ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,  in sanitize_ptr_alu()
13124 					env->insn_idx);  in sanitize_ptr_alu()
13130 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)  in sanitize_mark_insn_seen()  argument
13132 	struct bpf_verifier_state *vstate = env->cur_state;  in sanitize_mark_insn_seen()
13140 		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;  in sanitize_mark_insn_seen()
13143 static int sanitize_err(struct bpf_verifier_env *env,  in sanitize_err()  argument
13154 		verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",  in sanitize_err()
13158 		verbose(env, "R%d has pointer with unsupported alu operation, %s\n",  in sanitize_err()
13162 		verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",  in sanitize_err()
13166 		verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",  in sanitize_err()
13170 		verbose(env, "R%d could not be pushed for speculative verification, %s\n",  in sanitize_err()
13174 		verbose(env, "verifier internal error: unknown reason (%d)\n",  in sanitize_err()
13193 				struct bpf_verifier_env *env,  in check_stack_access_for_ptr_arithmetic()  argument
13202 		verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",  in check_stack_access_for_ptr_arithmetic()
13208 		verbose(env, "R%d stack pointer arithmetic goes out of range, "  in check_stack_access_for_ptr_arithmetic()
13216 static int sanitize_check_bounds(struct bpf_verifier_env *env,  in sanitize_check_bounds()  argument
13225 	if (env->bypass_spec_v1)  in sanitize_check_bounds()
13230 		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,  in sanitize_check_bounds()
13235 		if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {  in sanitize_check_bounds()
13236 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "  in sanitize_check_bounds()
13251  * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
13253 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,  in adjust_ptr_min_max_vals()  argument
13258 	struct bpf_verifier_state *vstate = env->cur_state;  in adjust_ptr_min_max_vals()
13278 		__mark_reg_unknown(env, dst_reg);  in adjust_ptr_min_max_vals()
13284 		if (opcode == BPF_SUB && env->allow_ptr_leaks) {  in adjust_ptr_min_max_vals()
13285 			__mark_reg_unknown(env, dst_reg);  in adjust_ptr_min_max_vals()
13289 		verbose(env,  in adjust_ptr_min_max_vals()
13296 		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",  in adjust_ptr_min_max_vals()
13297 			dst, reg_type_str(env, ptr_reg->type));  in adjust_ptr_min_max_vals()
13325 		verbose(env, "R%d pointer arithmetic on %s prohibited\n",  in adjust_ptr_min_max_vals()
13326 			dst, reg_type_str(env, ptr_reg->type));  in adjust_ptr_min_max_vals()
13336 	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||  in adjust_ptr_min_max_vals()
13337 	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))  in adjust_ptr_min_max_vals()
13344 		ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,  in adjust_ptr_min_max_vals()
13347 			return sanitize_err(env, insn, ret, off_reg, dst_reg);  in adjust_ptr_min_max_vals()
13390 			dst_reg->id = ++env->id_gen;  in adjust_ptr_min_max_vals()
13398 			verbose(env, "R%d tried to subtract pointer from scalar\n",  in adjust_ptr_min_max_vals()
13407 			verbose(env, "R%d subtraction from stack pointer prohibited\n",  in adjust_ptr_min_max_vals()
13446 			dst_reg->id = ++env->id_gen;  in adjust_ptr_min_max_vals()
13456 		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",  in adjust_ptr_min_max_vals()
13461 		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",  in adjust_ptr_min_max_vals()
13466 	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))  in adjust_ptr_min_max_vals()
13469 	if (sanitize_check_bounds(env, insn, dst_reg) < 0)  in adjust_ptr_min_max_vals()
13472 		ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,  in adjust_ptr_min_max_vals()
13475 			return sanitize_err(env, insn, ret, off_reg, dst_reg);  in adjust_ptr_min_max_vals()
14060 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,  in adjust_scalar_min_max_vals()  argument
14070 		__mark_reg_unknown(env, dst_reg);  in adjust_scalar_min_max_vals()
14075 		ret = sanitize_val_alu(env, insn);  in adjust_scalar_min_max_vals()
14077 			return sanitize_err(env, insn, ret, NULL, NULL);  in adjust_scalar_min_max_vals()
14157 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,  in adjust_reg_min_max_vals()  argument
14160 	struct bpf_verifier_state *vstate = env->cur_state;  in adjust_reg_min_max_vals()
14172 		struct bpf_insn_aux_data *aux = cur_aux(env);  in adjust_reg_min_max_vals()
14196 				if (opcode == BPF_SUB && env->allow_ptr_leaks) {  in adjust_reg_min_max_vals()
14197 					mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_reg_min_max_vals()
14200 				verbose(env, "R%d pointer %s pointer prohibited\n",  in adjust_reg_min_max_vals()
14209 				err = mark_chain_precision(env, insn->dst_reg);  in adjust_reg_min_max_vals()
14212 				return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
14217 			err = mark_chain_precision(env, insn->src_reg);  in adjust_reg_min_max_vals()
14220 			return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
14224 			err = mark_chain_precision(env, insn->src_reg);  in adjust_reg_min_max_vals()
14236 			return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
14242 		print_verifier_state(env, state, true);  in adjust_reg_min_max_vals()
14243 		verbose(env, "verifier internal error: unexpected ptr_reg\n");  in adjust_reg_min_max_vals()
14247 		print_verifier_state(env, state, true);  in adjust_reg_min_max_vals()
14248 		verbose(env, "verifier internal error: no src_reg\n");  in adjust_reg_min_max_vals()
14251 	err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);  in adjust_reg_min_max_vals()
14263 	if (env->bpf_capable &&  in adjust_reg_min_max_vals()
14292 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_alu_op()  argument
14294 	struct bpf_reg_state *regs = cur_regs(env);  in check_alu_op()
14303 				verbose(env, "BPF_NEG uses reserved fields\n");  in check_alu_op()
14311 				verbose(env, "BPF_END uses reserved fields\n");  in check_alu_op()
14317 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_alu_op()
14321 		if (is_pointer_value(env, insn->dst_reg)) {  in check_alu_op()
14322 			verbose(env, "R%d pointer arithmetic prohibited\n",  in check_alu_op()
14328 		err = check_reg_arg(env, insn->dst_reg, DST_OP);  in check_alu_op()
14338 					verbose(env, "BPF_MOV uses reserved fields\n");  in check_alu_op()
14343 					verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");  in check_alu_op()
14346 				if (!env->prog->aux->arena) {  in check_alu_op()
14347 …	verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n");  in check_alu_op()
14353 					verbose(env, "BPF_MOV uses reserved fields\n");  in check_alu_op()
14359 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_alu_op()
14364 				verbose(env, "BPF_MOV uses reserved fields\n");  in check_alu_op()
14370 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in check_alu_op()
14381 					mark_reg_unknown(env, regs, insn->dst_reg);  in check_alu_op()
14385 						dst_reg->subreg_def = env->insn_idx + 1;  in check_alu_op()
14391 					assign_scalar_id_before_mov(env, src_reg);  in check_alu_op()
14397 					if (is_pointer_value(env, insn->src_reg)) {  in check_alu_op()
14398 						verbose(env,  in check_alu_op()
14407 							assign_scalar_id_before_mov(env, src_reg);  in check_alu_op()
14415 						mark_reg_unknown(env, regs, insn->dst_reg);  in check_alu_op()
14420 				if (is_pointer_value(env, insn->src_reg)) {  in check_alu_op()
14421 					verbose(env,  in check_alu_op()
14430 							assign_scalar_id_before_mov(env, src_reg);  in check_alu_op()
14439 						dst_reg->subreg_def = env->insn_idx + 1;  in check_alu_op()
14445 							assign_scalar_id_before_mov(env, src_reg);  in check_alu_op()
14450 						dst_reg->subreg_def = env->insn_idx + 1;  in check_alu_op()
14454 					mark_reg_unknown(env, regs,  in check_alu_op()
14465 			mark_reg_unknown(env, regs, insn->dst_reg);  in check_alu_op()
14477 		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);  in check_alu_op()
14485 				verbose(env, "BPF_ALU uses reserved fields\n");  in check_alu_op()
14489 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_alu_op()
14495 				verbose(env, "BPF_ALU uses reserved fields\n");  in check_alu_op()
14501 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_alu_op()
14507 			verbose(env, "div by zero\n");  in check_alu_op()
14516 				verbose(env, "invalid shift %d\n", insn->imm);  in check_alu_op()
14522 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in check_alu_op()
14523 		err = err ?: adjust_reg_min_max_vals(env, insn);  in check_alu_op()
14528 	return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");  in check_alu_op()
15049 static int reg_set_min_max(struct bpf_verifier_env *env,  in reg_set_min_max()  argument
15075 	err = reg_bounds_sanity_check(env, true_reg1, "true_reg1");  in reg_set_min_max()
15076 	err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2");  in reg_set_min_max()
15077 	err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1");  in reg_set_min_max()
15078 	err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2");  in reg_set_min_max()
15350 static int check_cond_jmp_op(struct bpf_verifier_env *env,  in check_cond_jmp_op()  argument
15353 	struct bpf_verifier_state *this_branch = env->cur_state;  in check_cond_jmp_op()
15366 		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);  in check_cond_jmp_op()
15371 		struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;  in check_cond_jmp_op()
15377 			verbose(env, "invalid may_goto off %d imm %d\n",  in check_cond_jmp_op()
15381 		prev_st = find_prev_entry(env, cur_st->parent, idx);  in check_cond_jmp_op()
15384 		queued_st = push_stack(env, idx + 1, idx, false);  in check_cond_jmp_op()
15390 			widen_imprecise_scalars(env, prev_st, queued_st);  in check_cond_jmp_op()
15396 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_cond_jmp_op()
15403 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");  in check_cond_jmp_op()
15408 		err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_cond_jmp_op()
15414 		    is_pointer_value(env, insn->src_reg)) {  in check_cond_jmp_op()
15415 			verbose(env, "R%d pointer comparison prohibited\n",  in check_cond_jmp_op()
15421 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");  in check_cond_jmp_op()
15424 		src_reg = &env->fake_reg[0];  in check_cond_jmp_op()
15437 			err = mark_chain_precision(env, insn->dst_reg);  in check_cond_jmp_op()
15440 			err = mark_chain_precision(env, insn->src_reg);  in check_cond_jmp_op()
15450 		if (!env->bypass_spec_v1 &&  in check_cond_jmp_op()
15451 		    !sanitize_speculative_path(env, insn, *insn_idx + 1,  in check_cond_jmp_op()
15454 		if (env->log.level & BPF_LOG_LEVEL)  in check_cond_jmp_op()
15455 			print_insn_state(env, this_branch->frame[this_branch->curframe]);  in check_cond_jmp_op()
15463 		if (!env->bypass_spec_v1 &&  in check_cond_jmp_op()
15464 		    !sanitize_speculative_path(env, insn,  in check_cond_jmp_op()
15468 		if (env->log.level & BPF_LOG_LEVEL)  in check_cond_jmp_op()
15469 			print_insn_state(env, this_branch->frame[this_branch->curframe]);  in check_cond_jmp_op()
15483 		err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs));  in check_cond_jmp_op()
15488 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,  in check_cond_jmp_op()
15495 		err = reg_set_min_max(env,  in check_cond_jmp_op()
15504 		memcpy(&env->fake_reg[1], &env->fake_reg[0],  in check_cond_jmp_op()
15505 		       sizeof(env->fake_reg[0]));  in check_cond_jmp_op()
15506 		err = reg_set_min_max(env,  in check_cond_jmp_op()
15508 				      &env->fake_reg[0],  in check_cond_jmp_op()
15509 				      dst_reg, &env->fake_reg[1],  in check_cond_jmp_op()
15580 		   is_pointer_value(env, insn->dst_reg)) {  in check_cond_jmp_op()
15581 		verbose(env, "R%d pointer comparison prohibited\n",  in check_cond_jmp_op()
15585 	if (env->log.level & BPF_LOG_LEVEL)  in check_cond_jmp_op()
15586 		print_insn_state(env, this_branch->frame[this_branch->curframe]);  in check_cond_jmp_op()
15591 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_imm()  argument
15593 	struct bpf_insn_aux_data *aux = cur_aux(env);  in check_ld_imm()
15594 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_imm()
15600 		verbose(env, "invalid BPF_LD_IMM insn\n");  in check_ld_imm()
15604 		verbose(env, "BPF_LD_IMM64 uses reserved fields\n");  in check_ld_imm()
15608 	err = check_reg_arg(env, insn->dst_reg, DST_OP);  in check_ld_imm()
15625 	mark_reg_known_zero(env, regs, insn->dst_reg);  in check_ld_imm()
15638 			verbose(env, "bpf verifier is misconfigured\n");  in check_ld_imm()
15645 		struct bpf_prog_aux *aux = env->prog->aux;  in check_ld_imm()
15646 		u32 subprogno = find_subprog(env,  in check_ld_imm()
15647 					     env->insn_idx + insn->imm + 1);  in check_ld_imm()
15650 			verbose(env, "missing btf func_info\n");  in check_ld_imm()
15654 			verbose(env, "callback function not static\n");  in check_ld_imm()
15663 	map = env->used_maps[aux->map_index];  in check_ld_imm()
15669 			__mark_reg_unknown(env, dst_reg);  in check_ld_imm()
15680 		verbose(env, "bpf verifier is misconfigured\n");  in check_ld_imm()
15714 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_abs()  argument
15716 	struct bpf_reg_state *regs = cur_regs(env);  in check_ld_abs()
15721 	if (!may_access_skb(resolve_prog_type(env->prog))) {  in check_ld_abs()
15722 		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");  in check_ld_abs()
15726 	if (!env->ops->gen_ld_abs) {  in check_ld_abs()
15727 		verbose(env, "bpf verifier is misconfigured\n");  in check_ld_abs()
15734 		verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");  in check_ld_abs()
15739 	err = check_reg_arg(env, ctx_reg, SRC_OP);  in check_ld_abs()
15747 	err = check_reference_leak(env, false);  in check_ld_abs()
15749 		verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");  in check_ld_abs()
15753 	if (env->cur_state->active_lock.ptr) {  in check_ld_abs()
15754 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");  in check_ld_abs()
15758 	if (env->cur_state->active_rcu_lock) {  in check_ld_abs()
15759 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n");  in check_ld_abs()
15763 	if (env->cur_state->active_preempt_lock) {  in check_ld_abs()
15764 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_preempt_disable-ed region\n");  in check_ld_abs()
15769 		verbose(env,  in check_ld_abs()
15776 		err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_ld_abs()
15781 	err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);  in check_ld_abs()
15787 		mark_reg_not_init(env, regs, caller_saved[i]);  in check_ld_abs()
15788 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);  in check_ld_abs()
15795 	mark_reg_unknown(env, regs, BPF_REG_0);  in check_ld_abs()
15797 	regs[BPF_REG_0].subreg_def = env->insn_idx + 1;  in check_ld_abs()
15801 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)  in check_return_code()  argument
15805 	const struct bpf_prog *prog = env->prog;  in check_return_code()
15808 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);  in check_return_code()
15810 	struct bpf_func_state *frame = env->cur_state->frame[0];  in check_return_code()
15837 	err = check_reg_arg(env, regno, SRC_OP);  in check_return_code()
15841 	if (is_pointer_value(env, regno)) {  in check_return_code()
15842 		verbose(env, "R%d leaks addr as return value\n", regno);  in check_return_code()
15846 	reg = cur_regs(env) + regno;  in check_return_code()
15857 			verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",  in check_return_code()
15858 				regno, reg_type_str(env, reg->type));  in check_return_code()
15866 		if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||  in check_return_code()
15867 		    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||  in check_return_code()
15868 		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG ||  in check_return_code()
15869 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||  in check_return_code()
15870 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||  in check_return_code()
15871 		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME ||  in check_return_code()
15872 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||  in check_return_code()
15873 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME ||  in check_return_code()
15874 		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME)  in check_return_code()
15876 		if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||  in check_return_code()
15877 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)  in check_return_code()
15881 		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {  in check_return_code()
15893 		if (!env->prog->aux->attach_btf_id)  in check_return_code()
15898 		switch (env->prog->expected_attach_type) {  in check_return_code()
15917 		if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {  in check_return_code()
15919 			if (!get_func_retval_range(env->prog, &range))  in check_return_code()
15925 		} else if (!env->prog->aux->attach_func_proto->type) {  in check_return_code()
15946 		verbose(env, "%s the register R%d is not a known value (%s)\n",  in check_return_code()
15947 			exit_ctx, regno, reg_type_str(env, reg->type));  in check_return_code()
15951 	err = mark_chain_precision(env, regno);  in check_return_code()
15956 		verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);  in check_return_code()
15961 			verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");  in check_return_code()
15967 		env->prog->enforce_expected_attach_type = 1;  in check_return_code()
16011 static void mark_prune_point(struct bpf_verifier_env *env, int idx)  in mark_prune_point()  argument
16013 	env->insn_aux_data[idx].prune_point = true;  in mark_prune_point()
16016 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx)  in is_prune_point()  argument
16018 	return env->insn_aux_data[insn_idx].prune_point;  in is_prune_point()
16021 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)  in mark_force_checkpoint()  argument
16023 	env->insn_aux_data[idx].force_checkpoint = true;  in mark_force_checkpoint()
16026 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)  in is_force_checkpoint()  argument
16028 	return env->insn_aux_data[insn_idx].force_checkpoint;  in is_force_checkpoint()
16031 static void mark_calls_callback(struct bpf_verifier_env *env, int idx)  in mark_calls_callback()  argument
16033 	env->insn_aux_data[idx].calls_callback = true;  in mark_calls_callback()
16036 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx)  in calls_callback()  argument
16038 	return env->insn_aux_data[insn_idx].calls_callback;  in calls_callback()
16051 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)  in push_insn()  argument
16053 	int *insn_stack = env->cfg.insn_stack;  in push_insn()
16054 	int *insn_state = env->cfg.insn_state;  in push_insn()
16062 	if (w < 0 || w >= env->prog->len) {  in push_insn()
16063 		verbose_linfo(env, t, "%d: ", t);  in push_insn()
16064 		verbose(env, "jump out of range from insn %d to %d\n", t, w);  in push_insn()
16070 		mark_prune_point(env, w);  in push_insn()
16071 		mark_jmp_point(env, w);  in push_insn()
16078 		if (env->cfg.cur_stack >= env->prog->len)  in push_insn()
16080 		insn_stack[env->cfg.cur_stack++] = w;  in push_insn()
16083 		if (env->bpf_capable)  in push_insn()
16085 		verbose_linfo(env, t, "%d: ", t);  in push_insn()
16086 		verbose_linfo(env, w, "%d: ", w);  in push_insn()
16087 		verbose(env, "back-edge from insn %d to %d\n", t, w);  in push_insn()
16093 		verbose(env, "insn state internal bug\n");  in push_insn()
16100 				struct bpf_verifier_env *env,  in visit_func_call_insn()  argument
16106 	ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);  in visit_func_call_insn()
16110 	mark_prune_point(env, t + insn_sz);  in visit_func_call_insn()
16112 	mark_jmp_point(env, t + insn_sz);  in visit_func_call_insn()
16115 		mark_prune_point(env, t);  in visit_func_call_insn()
16116 		ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);  in visit_func_call_insn()
16149 static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)  in verifier_inlines_helper_call()  argument
16154 		return env->prog->jit_requested && bpf_jit_supports_percpu_insn();  in verifier_inlines_helper_call()
16232  * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;
16233  * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;
16234  * - update env->subprog_info[*]->fastcall_stack_off to find an offset
16236  * - update env->subprog_info[*]->keep_fastcall_stack.
16258 static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,  in mark_fastcall_pattern_for_call()  argument
16262 	struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;  in mark_fastcall_pattern_for_call()
16263 	struct bpf_insn *call = &env->prog->insnsi[insn_idx];  in mark_fastcall_pattern_for_call()
16272 		if (get_helper_proto(env, call->imm, &fn) < 0)  in mark_fastcall_pattern_for_call()
16277 				 (verifier_inlines_helper_call(env, call->imm) ||  in mark_fastcall_pattern_for_call()
16285 		err = fetch_kfunc_meta(env, call, &meta, NULL);  in mark_fastcall_pattern_for_call()
16309 		if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)  in mark_fastcall_pattern_for_call()
16332 		env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;  in mark_fastcall_pattern_for_call()
16333 		env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;  in mark_fastcall_pattern_for_call()
16355 		env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;  in mark_fastcall_pattern_for_call()
16361 static int mark_fastcall_patterns(struct bpf_verifier_env *env)  in mark_fastcall_patterns()  argument
16363 	struct bpf_subprog_info *subprog = env->subprog_info;  in mark_fastcall_patterns()
16368 	for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {  in mark_fastcall_patterns()
16372 			insn = env->prog->insnsi + i;  in mark_fastcall_patterns()
16380 			insn = env->prog->insnsi + i;  in mark_fastcall_patterns()
16383 			mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);  in mark_fastcall_patterns()
16394 static int visit_insn(int t, struct bpf_verifier_env *env)  in visit_insn()  argument
16396 	struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];  in visit_insn()
16400 		return visit_func_call_insn(t, insns, env, true);  in visit_insn()
16406 		return push_insn(t, t + insn_sz, FALLTHROUGH, env);  in visit_insn()
16420 			mark_prune_point(env, t);  in visit_insn()
16431 			mark_calls_callback(env, t);  in visit_insn()
16432 			mark_force_checkpoint(env, t);  in visit_insn()
16433 			mark_prune_point(env, t);  in visit_insn()
16434 			mark_jmp_point(env, t);  in visit_insn()
16439 			ret = fetch_kfunc_meta(env, insn, &meta, NULL);  in visit_insn()
16441 				mark_prune_point(env, t);  in visit_insn()
16453 				mark_force_checkpoint(env, t);  in visit_insn()
16456 		return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);  in visit_insn()
16468 		ret = push_insn(t, t + off + 1, FALLTHROUGH, env);  in visit_insn()
16472 		mark_prune_point(env, t + off + 1);  in visit_insn()
16473 		mark_jmp_point(env, t + off + 1);  in visit_insn()
16479 		mark_prune_point(env, t);  in visit_insn()
16481 			mark_force_checkpoint(env, t);  in visit_insn()
16483 		ret = push_insn(t, t + 1, FALLTHROUGH, env);  in visit_insn()
16487 		return push_insn(t, t + insn->off + 1, BRANCH, env);  in visit_insn()
16494 static int check_cfg(struct bpf_verifier_env *env)  in check_cfg()  argument
16496 	int insn_cnt = env->prog->len;  in check_cfg()
16501 	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);  in check_cfg()
16505 	insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);  in check_cfg()
16513 	env->cfg.cur_stack = 1;  in check_cfg()
16516 	while (env->cfg.cur_stack > 0) {  in check_cfg()
16517 		int t = insn_stack[env->cfg.cur_stack - 1];  in check_cfg()
16519 		ret = visit_insn(t, env);  in check_cfg()
16523 			env->cfg.cur_stack--;  in check_cfg()
16529 				verbose(env, "visit_insn internal bug\n");  in check_cfg()
16536 	if (env->cfg.cur_stack < 0) {  in check_cfg()
16537 		verbose(env, "pop stack internal bug\n");  in check_cfg()
16542 	if (env->exception_callback_subprog && !ex_done) {  in check_cfg()
16543 		ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start;  in check_cfg()
16547 		env->cfg.cur_stack = 1;  in check_cfg()
16553 		struct bpf_insn *insn = &env->prog->insnsi[i];  in check_cfg()
16556 			verbose(env, "unreachable insn %d\n", i);  in check_cfg()
16562 				verbose(env, "jump into the middle of ldimm64 insn %d\n", i);  in check_cfg()
16574 	env->cfg.insn_state = env->cfg.insn_stack = NULL;  in check_cfg()
16578 static int check_abnormal_return(struct bpf_verifier_env *env)  in check_abnormal_return()  argument
16582 	for (i = 1; i < env->subprog_cnt; i++) {  in check_abnormal_return()
16583 		if (env->subprog_info[i].has_ld_abs) {  in check_abnormal_return()
16584 			verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");  in check_abnormal_return()
16587 		if (env->subprog_info[i].has_tail_call) {  in check_abnormal_return()
16588 			verbose(env, "tail_call is not allowed in subprogs without BTF\n");  in check_abnormal_return()
16599 static int check_btf_func_early(struct bpf_verifier_env *env,  in check_btf_func_early()  argument
16615 		if (check_abnormal_return(env))  in check_btf_func_early()
16624 		verbose(env, "invalid func info rec size %u\n", urec_size);  in check_btf_func_early()
16628 	prog = env->prog;  in check_btf_func_early()
16642 				verbose(env, "nonzero tailing record in func info");  in check_btf_func_early()
16663 				verbose(env,  in check_btf_func_early()
16669 			verbose(env,  in check_btf_func_early()
16678 			verbose(env, "invalid type id %d in func info",  in check_btf_func_early()
16701 static int check_btf_func(struct bpf_verifier_env *env,  in check_btf_func()  argument
16717 		if (check_abnormal_return(env))  in check_btf_func()
16721 	if (nfuncs != env->subprog_cnt) {  in check_btf_func()
16722 		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");  in check_btf_func()
16728 	prog = env->prog;  in check_btf_func()
16742 		if (env->subprog_info[i].start != krecord[i].insn_off) {  in check_btf_func()
16743 			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");  in check_btf_func()
16756 		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {  in check_btf_func()
16757 			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");  in check_btf_func()
16760 		if (i && !scalar_return && env->subprog_info[i].has_tail_call) {  in check_btf_func()
16761 			verbose(env, "tail_call is only allowed in functions that return 'int'.\n");  in check_btf_func()
16776 static void adjust_btf_func(struct bpf_verifier_env *env)  in adjust_btf_func()  argument
16778 	struct bpf_prog_aux *aux = env->prog->aux;  in adjust_btf_func()
16785 	for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++)  in adjust_btf_func()
16786 		aux->func_info[i].insn_off = env->subprog_info[i].start;  in adjust_btf_func()
16792 static int check_btf_line(struct bpf_verifier_env *env,  in check_btf_line()  argument
16824 	prog = env->prog;  in check_btf_line()
16828 	sub = env->subprog_info;  in check_btf_line()
16836 				verbose(env, "nonzero tailing record in line_info");  in check_btf_line()
16863 			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",  in check_btf_line()
16871 			verbose(env,  in check_btf_line()
16880 			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);  in check_btf_line()
16885 		if (s != env->subprog_cnt) {  in check_btf_line()
16890 				verbose(env, "missing bpf_line_info for func#%u\n", s);  in check_btf_line()
16900 	if (s != env->subprog_cnt) {  in check_btf_line()
16901 		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",  in check_btf_line()
16902 			env->subprog_cnt - s, s);  in check_btf_line()
16920 static int check_core_relo(struct bpf_verifier_env *env,  in check_core_relo()  argument
16926 	struct bpf_prog *prog = env->prog;  in check_core_relo()
16929 		.log = &env->log,  in check_core_relo()
16959 				verbose(env, "nonzero tailing record in core_relo");  in check_core_relo()
16974 			verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",  in check_core_relo()
16989 static int check_btf_info_early(struct bpf_verifier_env *env,  in check_btf_info_early()  argument
16997 		if (check_abnormal_return(env))  in check_btf_info_early()
17009 	env->prog->aux->btf = btf;  in check_btf_info_early()
17011 	err = check_btf_func_early(env, attr, uattr);  in check_btf_info_early()
17017 static int check_btf_info(struct bpf_verifier_env *env,  in check_btf_info()  argument
17024 		if (check_abnormal_return(env))  in check_btf_info()
17029 	err = check_btf_func(env, attr, uattr);  in check_btf_info()
17033 	err = check_btf_line(env, attr, uattr);  in check_btf_info()
17037 	err = check_core_relo(env, attr, uattr);  in check_btf_info()
17109 static void clean_func_state(struct bpf_verifier_env *env,  in clean_func_state()  argument
17123 			__mark_reg_not_init(env, &st->regs[i]);  in clean_func_state()
17131 			__mark_reg_not_init(env, &st->stack[i].spilled_ptr);  in clean_func_state()
17138 static void clean_verifier_state(struct bpf_verifier_env *env,  in clean_verifier_state()  argument
17148 		clean_func_state(env, st->frame[i]);  in clean_verifier_state()
17183 static void clean_live_states(struct bpf_verifier_env *env, int insn,  in clean_live_states()  argument
17188 	sl = *explored_state(env, insn);  in clean_live_states()
17195 		clean_verifier_state(env, &sl->state);  in clean_live_states()
17217 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,  in regsafe()  argument
17259 		if (env->explore_alu_limits) {  in regsafe()
17355 static bool is_stack_all_misc(struct bpf_verifier_env *env,  in is_stack_all_misc()  argument
17362 		    (stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack))  in is_stack_all_misc()
17370 static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,  in scalar_reg_for_stack()  argument
17376 	if (is_stack_all_misc(env, stack))  in scalar_reg_for_stack()
17382 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,  in stacksafe()  argument
17413 		if (env->allow_uninit_stack &&  in stacksafe()
17428 		old_reg = scalar_reg_for_stack(env, &old->stack[spi]);  in stacksafe()
17429 		cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]);  in stacksafe()
17431 			if (!regsafe(env, old_reg, cur_reg, idmap, exact))  in stacksafe()
17467 			if (!regsafe(env, &old->stack[spi].spilled_ptr,  in stacksafe()
17549 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,  in func_states_equal()  argument
17558 		if (!regsafe(env, &old->regs[i], &cur->regs[i],  in func_states_equal()
17559 			     &env->idmap_scratch, exact))  in func_states_equal()
17562 	if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))  in func_states_equal()
17565 	if (!refsafe(old, cur, &env->idmap_scratch))  in func_states_equal()
17571 static void reset_idmap_scratch(struct bpf_verifier_env *env)  in reset_idmap_scratch()  argument
17573 	env->idmap_scratch.tmp_id_gen = env->id_gen;  in reset_idmap_scratch()
17574 	memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map));  in reset_idmap_scratch()
17577 static bool states_equal(struct bpf_verifier_env *env,  in states_equal()  argument
17587 	reset_idmap_scratch(env);  in states_equal()
17605 	    !check_ids(old->active_lock.id, cur->active_lock.id, &env->idmap_scratch))  in states_equal()
17623 		if (!func_states_equal(env, old->frame[i], cur->frame[i], exact))  in states_equal()
17632 static int propagate_liveness_reg(struct bpf_verifier_env *env,  in propagate_liveness_reg()  argument
17651 	err = mark_reg_read(env, reg, parent_reg, flag);  in propagate_liveness_reg()
17665 static int propagate_liveness(struct bpf_verifier_env *env,  in propagate_liveness()  argument
17687 			err = propagate_liveness_reg(env, &state_reg[i],  in propagate_liveness()
17692 				mark_insn_zext(env, &parent_reg[i]);  in propagate_liveness()
17700 			err = propagate_liveness_reg(env, state_reg,  in propagate_liveness()
17712 static int propagate_precision(struct bpf_verifier_env *env,  in propagate_precision()  argument
17729 			if (env->log.level & BPF_LOG_LEVEL2) {  in propagate_precision()
17731 					verbose(env, "frame %d: propagating r%d", fr, i);  in propagate_precision()
17733 					verbose(env, ",r%d", i);  in propagate_precision()
17735 			bt_set_frame_reg(&env->bt, fr, i);  in propagate_precision()
17747 			if (env->log.level & BPF_LOG_LEVEL2) {  in propagate_precision()
17749 					verbose(env, "frame %d: propagating fp%d",  in propagate_precision()
17752 					verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE);  in propagate_precision()
17754 			bt_set_frame_slot(&env->bt, fr, i);  in propagate_precision()
17758 			verbose(env, "\n");  in propagate_precision()
17761 	err = mark_chain_precision_batch(env);  in propagate_precision()
17786 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)  in is_iter_next_insn()  argument
17788 	return env->insn_aux_data[insn_idx].is_iter_next;  in is_iter_next_insn()
17873 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)  in is_state_visited()  argument
17877 	struct bpf_verifier_state *cur = env->cur_state, *new, *loop_entry;  in is_state_visited()
17881 	force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||  in is_state_visited()
17894 	if (env->jmps_processed - env->prev_jmps_processed >= 2 &&  in is_state_visited()
17895 	    env->insn_processed - env->prev_insn_processed >= 8)  in is_state_visited()
17898 	pprev = explored_state(env, insn_idx);  in is_state_visited()
17901 	clean_live_states(env, insn_idx, cur);  in is_state_visited()
17962 			if (is_iter_next_insn(env, insn_idx)) {  in is_state_visited()
17963 				if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {  in is_state_visited()
17978 					iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;  in is_state_visited()
17986 			if (is_may_goto_insn_at(env, insn_idx)) {  in is_state_visited()
17988 				    states_equal(env, &sl->state, cur, RANGE_WITHIN)) {  in is_state_visited()
17993 			if (calls_callback(env, insn_idx)) {  in is_state_visited()
17994 				if (states_equal(env, &sl->state, cur, RANGE_WITHIN))  in is_state_visited()
18000 			    states_equal(env, &sl->state, cur, EXACT) &&  in is_state_visited()
18004 				verbose_linfo(env, insn_idx, "; ");  in is_state_visited()
18005 				verbose(env, "infinite loop detected at insn %d\n", insn_idx);  in is_state_visited()
18006 				verbose(env, "cur state:");  in is_state_visited()
18007 				print_verifier_state(env, cur->frame[cur->curframe], true);  in is_state_visited()
18008 				verbose(env, "old state:");  in is_state_visited()
18009 				print_verifier_state(env, sl->state.frame[cur->curframe], true);  in is_state_visited()
18026 			    env->jmps_processed - env->prev_jmps_processed < 20 &&  in is_state_visited()
18027 			    env->insn_processed - env->prev_insn_processed < 100)  in is_state_visited()
18058 		if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) {  in is_state_visited()
18066 			 * If we have any write marks in env->cur_state, they  in is_state_visited()
18073 			err = propagate_liveness(env, &sl->state, cur);  in is_state_visited()
18080 			if (is_jmp_point(env, env->insn_idx))  in is_state_visited()
18081 				err = err ? : push_jmp_history(env, cur, 0, 0);  in is_state_visited()
18082 			err = err ? : propagate_precision(env, &sl->state);  in is_state_visited()
18104 		n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;  in is_state_visited()
18119 				env->peak_states--;  in is_state_visited()
18125 				sl->next = env->free_list;  in is_state_visited()
18126 				env->free_list = sl;  in is_state_visited()
18136 	if (env->max_states_per_insn < states_cnt)  in is_state_visited()
18137 		env->max_states_per_insn = states_cnt;  in is_state_visited()
18139 	if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)  in is_state_visited()
18157 	env->total_states++;  in is_state_visited()
18158 	env->peak_states++;  in is_state_visited()
18159 	env->prev_jmps_processed = env->jmps_processed;  in is_state_visited()
18160 	env->prev_insn_processed = env->insn_processed;  in is_state_visited()
18163 	if (env->bpf_capable)  in is_state_visited()
18164 		mark_all_scalars_imprecise(env, cur);  in is_state_visited()
18182 	new_sl->next = *explored_state(env, insn_idx);  in is_state_visited()
18183 	*explored_state(env, insn_idx) = new_sl;  in is_state_visited()
18253 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,  in save_aux_ptr_type()  argument
18256 	enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;  in save_aux_ptr_type()
18283 			verbose(env, "same insn cannot be used with different pointers\n");  in save_aux_ptr_type()
18291 static int do_check(struct bpf_verifier_env *env)  in do_check()  argument
18293 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);  in do_check()
18294 	struct bpf_verifier_state *state = env->cur_state;  in do_check()
18295 	struct bpf_insn *insns = env->prog->insnsi;  in do_check()
18297 	int insn_cnt = env->prog->len;  in do_check()
18308 		env->cur_hist_ent = NULL;  in do_check()
18310 		env->prev_insn_idx = prev_insn_idx;  in do_check()
18311 		if (env->insn_idx >= insn_cnt) {  in do_check()
18312 			verbose(env, "invalid insn idx %d insn_cnt %d\n",  in do_check()
18313 				env->insn_idx, insn_cnt);  in do_check()
18317 		insn = &insns[env->insn_idx];  in do_check()
18320 		if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {  in do_check()
18321 			verbose(env,  in do_check()
18323 				env->insn_processed);  in do_check()
18327 		state->last_insn_idx = env->prev_insn_idx;  in do_check()
18329 		if (is_prune_point(env, env->insn_idx)) {  in do_check()
18330 			err = is_state_visited(env, env->insn_idx);  in do_check()
18335 				if (env->log.level & BPF_LOG_LEVEL) {  in do_check()
18337 						verbose(env, "\nfrom %d to %d%s: safe\n",  in do_check()
18338 							env->prev_insn_idx, env->insn_idx,  in do_check()
18339 							env->cur_state->speculative ?  in do_check()
18342 						verbose(env, "%d: safe\n", env->insn_idx);  in do_check()
18348 		if (is_jmp_point(env, env->insn_idx)) {  in do_check()
18349 			err = push_jmp_history(env, state, 0, 0);  in do_check()
18360 		if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {  in do_check()
18361 			verbose(env, "\nfrom %d to %d%s:",  in do_check()
18362 				env->prev_insn_idx, env->insn_idx,  in do_check()
18363 				env->cur_state->speculative ?  in do_check()
18365 			print_verifier_state(env, state->frame[state->curframe], true);  in do_check()
18369 		if (env->log.level & BPF_LOG_LEVEL) {  in do_check()
18373 				.private_data	= env,  in do_check()
18376 			if (verifier_state_scratched(env))  in do_check()
18377 				print_insn_state(env, state->frame[state->curframe]);  in do_check()
18379 			verbose_linfo(env, env->insn_idx, "; ");  in do_check()
18380 			env->prev_log_pos = env->log.end_pos;  in do_check()
18381 			verbose(env, "%d: ", env->insn_idx);  in do_check()
18382 			print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);  in do_check()
18383 			env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;  in do_check()
18384 			env->prev_log_pos = env->log.end_pos;  in do_check()
18387 		if (bpf_prog_is_offloaded(env->prog->aux)) {  in do_check()
18388 			err = bpf_prog_offload_verify_insn(env, env->insn_idx,  in do_check()
18389 							   env->prev_insn_idx);  in do_check()
18394 		regs = cur_regs(env);  in do_check()
18395 		sanitize_mark_insn_seen(env);  in do_check()
18396 		prev_insn_idx = env->insn_idx;  in do_check()
18399 			err = check_alu_op(env, insn);  in do_check()
18409 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in do_check()
18413 			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in do_check()
18422 			err = check_mem_access(env, env->insn_idx, insn->src_reg,  in do_check()
18426 			err = err ?: save_aux_ptr_type(env, src_reg_type, true);  in do_check()
18427 			err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], "ldx");  in do_check()
18434 				err = check_atomic(env, env->insn_idx, insn);  in do_check()
18437 				env->insn_idx++;  in do_check()
18442 				verbose(env, "BPF_STX uses reserved fields\n");  in do_check()
18447 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in do_check()
18451 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in do_check()
18458 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in do_check()
18464 			err = save_aux_ptr_type(env, dst_reg_type, false);  in do_check()
18472 				verbose(env, "BPF_ST uses reserved fields\n");  in do_check()
18476 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in do_check()
18483 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in do_check()
18489 			err = save_aux_ptr_type(env, dst_reg_type, false);  in do_check()
18495 			env->jmps_processed++;  in do_check()
18505 					verbose(env, "BPF_CALL uses reserved fields\n");  in do_check()
18509 				if (env->cur_state->active_lock.ptr) {  in do_check()
18513 						verbose(env, "function calls are not allowed while holding a lock\n");  in do_check()
18518 					err = check_func_call(env, insn, &env->insn_idx);  in do_check()
18520 					err = check_kfunc_call(env, insn, &env->insn_idx);  in do_check()
18526 					err = check_helper_call(env, insn, &env->insn_idx);  in do_check()
18531 				mark_reg_scratched(env, BPF_REG_0);  in do_check()
18538 					verbose(env, "BPF_JA uses reserved fields\n");  in do_check()
18543 					env->insn_idx += insn->off + 1;  in do_check()
18545 					env->insn_idx += insn->imm + 1;  in do_check()
18554 					verbose(env, "BPF_EXIT uses reserved fields\n");  in do_check()
18558 				if (env->cur_state->active_lock.ptr && !env->cur_state->curframe) {  in do_check()
18559 					verbose(env, "bpf_spin_unlock is missing\n");  in do_check()
18563 				if (env->cur_state->active_rcu_lock && !env->cur_state->curframe) {  in do_check()
18564 					verbose(env, "bpf_rcu_read_unlock is missing\n");  in do_check()
18568 				if (env->cur_state->active_preempt_lock && !env->cur_state->curframe) {  in do_check()
18569 					verbose(env, "%d bpf_preempt_enable%s missing\n",  in do_check()
18570 						env->cur_state->active_preempt_lock,  in do_check()
18571 						env->cur_state->active_preempt_lock == 1 ? " is" : "(s) are");  in do_check()
18581 				err = check_reference_leak(env, exception_exit);  in do_check()
18600 					err = prepare_func_exit(env, &env->insn_idx);  in do_check()
18607 				err = check_return_code(env, BPF_REG_0, "R0");  in do_check()
18611 				mark_verifier_state_scratched(env);  in do_check()
18612 				update_branch_counts(env, env->cur_state);  in do_check()
18613 				err = pop_stack(env, &prev_insn_idx,  in do_check()
18614 						&env->insn_idx, pop_log);  in do_check()
18624 				err = check_cond_jmp_op(env, insn, &env->insn_idx);  in do_check()
18632 				err = check_ld_abs(env, insn);  in do_check()
18637 				err = check_ld_imm(env, insn);  in do_check()
18641 				env->insn_idx++;  in do_check()
18642 				sanitize_mark_insn_seen(env);  in do_check()
18644 				verbose(env, "invalid BPF_LD mode\n");  in do_check()
18648 			verbose(env, "unknown insn class %d\n", class);  in do_check()
18652 		env->insn_idx++;  in do_check()
18689 static int check_pseudo_btf_id(struct bpf_verifier_env *env,  in check_pseudo_btf_id()  argument
18709 			verbose(env, "invalid module BTF object FD specified.\n");  in check_pseudo_btf_id()
18714 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"…  in check_pseudo_btf_id()
18723 		verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);  in check_pseudo_btf_id()
18729 		verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);  in check_pseudo_btf_id()
18737 		verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",  in check_pseudo_btf_id()
18777 			verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",  in check_pseudo_btf_id()
18791 	for (i = 0; i < env->used_btf_cnt; i++) {  in check_pseudo_btf_id()
18792 		if (env->used_btfs[i].btf == btf) {  in check_pseudo_btf_id()
18798 	if (env->used_btf_cnt >= MAX_USED_BTFS) {  in check_pseudo_btf_id()
18803 	btf_mod = &env->used_btfs[env->used_btf_cnt];  in check_pseudo_btf_id()
18816 	env->used_btf_cnt++;  in check_pseudo_btf_id()
18838 static int check_map_prog_compatibility(struct bpf_verifier_env *env,  in check_map_prog_compatibility()  argument
18848 			verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");  in check_map_prog_compatibility()
18855 			verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");  in check_map_prog_compatibility()
18860 			verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");  in check_map_prog_compatibility()
18867 			verbose(env, "tracing progs cannot use bpf_timer yet\n");  in check_map_prog_compatibility()
18874 			verbose(env, "tracing progs cannot use bpf_wq yet\n");  in check_map_prog_compatibility()
18881 		verbose(env, "offload device mismatch between prog and map\n");  in check_map_prog_compatibility()
18886 		verbose(env, "bpf_struct_ops map cannot be used in prog\n");  in check_map_prog_compatibility()
18911 			verbose(env,  in check_map_prog_compatibility()
18930 static int add_used_map_from_fd(struct bpf_verifier_env *env, int fd, bool *reused)  in add_used_map_from_fd()  argument
18938 		verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);  in add_used_map_from_fd()
18943 	for (i = 0; i < env->used_map_cnt; i++) {  in add_used_map_from_fd()
18944 		if (env->used_maps[i] == map) {  in add_used_map_from_fd()
18950 	if (env->used_map_cnt >= MAX_USED_MAPS) {  in add_used_map_from_fd()
18951 		verbose(env, "The total number of maps per program has reached the limit of %u\n",  in add_used_map_from_fd()
18956 	if (env->prog->sleepable)  in add_used_map_from_fd()
18967 	env->used_maps[env->used_map_cnt++] = map;  in add_used_map_from_fd()
18969 	return env->used_map_cnt - 1;  in add_used_map_from_fd()
18979 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)  in resolve_pseudo_ldimm64()  argument
18981 	struct bpf_insn *insn = env->prog->insnsi;  in resolve_pseudo_ldimm64()
18982 	int insn_cnt = env->prog->len;  in resolve_pseudo_ldimm64()
18985 	err = bpf_prog_calc_tag(env->prog);  in resolve_pseudo_ldimm64()
18993 			verbose(env, "BPF_LDX uses reserved fields\n");  in resolve_pseudo_ldimm64()
19008 				verbose(env, "invalid bpf_ld_imm64 insn\n");  in resolve_pseudo_ldimm64()
19017 				aux = &env->insn_aux_data[i];  in resolve_pseudo_ldimm64()
19018 				err = check_pseudo_btf_id(env, insn, aux);  in resolve_pseudo_ldimm64()
19025 				aux = &env->insn_aux_data[i];  in resolve_pseudo_ldimm64()
19043 				verbose(env, "unrecognized bpf_ld_imm64 insn\n");  in resolve_pseudo_ldimm64()
19050 				if (bpfptr_is_null(env->fd_array)) {  in resolve_pseudo_ldimm64()
19051 					verbose(env, "fd_idx without fd_array is invalid\n");  in resolve_pseudo_ldimm64()
19054 				if (copy_from_bpfptr_offset(&fd, env->fd_array,  in resolve_pseudo_ldimm64()
19064 			map_idx = add_used_map_from_fd(env, fd, &reused);  in resolve_pseudo_ldimm64()
19067 			map = env->used_maps[map_idx];  in resolve_pseudo_ldimm64()
19069 			aux = &env->insn_aux_data[i];  in resolve_pseudo_ldimm64()
19072 			err = check_map_prog_compatibility(env, map, env->prog);  in resolve_pseudo_ldimm64()
19083 					verbose(env, "direct value offset of %u is not allowed\n", off);  in resolve_pseudo_ldimm64()
19088 					verbose(env, "no direct value access support for this map type\n");  in resolve_pseudo_ldimm64()
19094 					verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",  in resolve_pseudo_ldimm64()
19111 			    bpf_cgroup_storage_assign(env->prog->aux, map)) {  in resolve_pseudo_ldimm64()
19112 				verbose(env, "only one cgroup storage of each type is allowed\n");  in resolve_pseudo_ldimm64()
19116 				if (env->prog->aux->arena) {  in resolve_pseudo_ldimm64()
19117 					verbose(env, "Only one arena per program\n");  in resolve_pseudo_ldimm64()
19120 				if (!env->allow_ptr_leaks || !env->bpf_capable) {  in resolve_pseudo_ldimm64()
19121 					verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");  in resolve_pseudo_ldimm64()
19124 				if (!env->prog->jit_requested) {  in resolve_pseudo_ldimm64()
19125 					verbose(env, "JIT is required to use arena\n");  in resolve_pseudo_ldimm64()
19129 					verbose(env, "JIT doesn't support arena\n");  in resolve_pseudo_ldimm64()
19132 				env->prog->aux->arena = (void *)map;  in resolve_pseudo_ldimm64()
19133 				if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {  in resolve_pseudo_ldimm64()
19134 					verbose(env, "arena's user address must be set via map_extra or mmap()\n");  in resolve_pseudo_ldimm64()
19147 			verbose(env, "unknown opcode %02x\n", insn->code);  in resolve_pseudo_ldimm64()
19160 static void release_maps(struct bpf_verifier_env *env)  in release_maps()  argument
19162 	__bpf_free_used_maps(env->prog->aux, env->used_maps,  in release_maps()
19163 			     env->used_map_cnt);  in release_maps()
19167 static void release_btfs(struct bpf_verifier_env *env)  in release_btfs()  argument
19169 	__bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);  in release_btfs()
19173 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)  in convert_pseudo_ld_imm64()  argument
19175 	struct bpf_insn *insn = env->prog->insnsi;  in convert_pseudo_ld_imm64()
19176 	int insn_cnt = env->prog->len;  in convert_pseudo_ld_imm64()
19188 /* single env->prog->insni[off] instruction was replaced with the range
19192 static void adjust_insn_aux_data(struct bpf_verifier_env *env,  in adjust_insn_aux_data()  argument
19196 	struct bpf_insn_aux_data *old_data = env->insn_aux_data;  in adjust_insn_aux_data()
19206 	old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);  in adjust_insn_aux_data()
19218 		new_data[i].zext_dst = insn_has_def32(env, insn + i);  in adjust_insn_aux_data()
19220 	env->insn_aux_data = new_data;  in adjust_insn_aux_data()
19224 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)  in adjust_subprog_starts()  argument
19231 	for (i = 0; i <= env->subprog_cnt; i++) {  in adjust_subprog_starts()
19232 		if (env->subprog_info[i].start <= off)  in adjust_subprog_starts()
19234 		env->subprog_info[i].start += len - 1;  in adjust_subprog_starts()
19252 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,  in bpf_patch_insn_data()  argument
19259 		new_data = vzalloc(array_size(env->prog->len + len - 1,  in bpf_patch_insn_data()
19265 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);  in bpf_patch_insn_data()
19268 			verbose(env,  in bpf_patch_insn_data()
19270 				env->insn_aux_data[off].orig_idx);  in bpf_patch_insn_data()
19274 	adjust_insn_aux_data(env, new_data, new_prog, off, len);  in bpf_patch_insn_data()
19275 	adjust_subprog_starts(env, off, len);  in bpf_patch_insn_data()
19318 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,  in adjust_subprog_starts_after_remove()  argument
19324 	for (i = 0; i < env->subprog_cnt; i++)  in adjust_subprog_starts_after_remove()
19325 		if (env->subprog_info[i].start >= off)  in adjust_subprog_starts_after_remove()
19328 	for (j = i; j < env->subprog_cnt; j++)  in adjust_subprog_starts_after_remove()
19329 		if (env->subprog_info[j].start >= off + cnt)  in adjust_subprog_starts_after_remove()
19334 	if (env->subprog_info[j].start != off + cnt)  in adjust_subprog_starts_after_remove()
19338 		struct bpf_prog_aux *aux = env->prog->aux;  in adjust_subprog_starts_after_remove()
19342 		move = env->subprog_cnt + 1 - j;  in adjust_subprog_starts_after_remove()
19344 		memmove(env->subprog_info + i,  in adjust_subprog_starts_after_remove()
19345 			env->subprog_info + j,  in adjust_subprog_starts_after_remove()
19346 			sizeof(*env->subprog_info) * move);  in adjust_subprog_starts_after_remove()
19347 		env->subprog_cnt -= j - i;  in adjust_subprog_starts_after_remove()
19363 		if (env->subprog_info[i].start == off)  in adjust_subprog_starts_after_remove()
19368 	for (; i <= env->subprog_cnt; i++)  in adjust_subprog_starts_after_remove()
19369 		env->subprog_info[i].start -= cnt;  in adjust_subprog_starts_after_remove()
19374 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,  in bpf_adj_linfo_after_remove()  argument
19377 	struct bpf_prog *prog = env->prog;  in bpf_adj_linfo_after_remove()
19424 	for (i = 0; i <= env->subprog_cnt; i++)  in bpf_adj_linfo_after_remove()
19425 		if (env->subprog_info[i].linfo_idx > l_off) {  in bpf_adj_linfo_after_remove()
19429 			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)  in bpf_adj_linfo_after_remove()
19430 				env->subprog_info[i].linfo_idx -= l_cnt;  in bpf_adj_linfo_after_remove()
19432 				env->subprog_info[i].linfo_idx = l_off;  in bpf_adj_linfo_after_remove()
19438 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)  in verifier_remove_insns()  argument
19440 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in verifier_remove_insns()
19441 	unsigned int orig_prog_len = env->prog->len;  in verifier_remove_insns()
19444 	if (bpf_prog_is_offloaded(env->prog->aux))  in verifier_remove_insns()
19445 		bpf_prog_offload_remove_insns(env, off, cnt);  in verifier_remove_insns()
19447 	err = bpf_remove_insns(env->prog, off, cnt);  in verifier_remove_insns()
19451 	err = adjust_subprog_starts_after_remove(env, off, cnt);  in verifier_remove_insns()
19455 	err = bpf_adj_linfo_after_remove(env, off, cnt);  in verifier_remove_insns()
19476 static void sanitize_dead_code(struct bpf_verifier_env *env)  in sanitize_dead_code()  argument
19478 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in sanitize_dead_code()
19480 	struct bpf_insn *insn = env->prog->insnsi;  in sanitize_dead_code()
19481 	const int insn_cnt = env->prog->len;  in sanitize_dead_code()
19506 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)  in opt_hard_wire_dead_code_branches()  argument
19508 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in opt_hard_wire_dead_code_branches()
19510 	struct bpf_insn *insn = env->prog->insnsi;  in opt_hard_wire_dead_code_branches()
19511 	const int insn_cnt = env->prog->len;  in opt_hard_wire_dead_code_branches()
19525 		if (bpf_prog_is_offloaded(env->prog->aux))  in opt_hard_wire_dead_code_branches()
19526 			bpf_prog_offload_replace_insn(env, i, &ja);  in opt_hard_wire_dead_code_branches()
19532 static int opt_remove_dead_code(struct bpf_verifier_env *env)  in opt_remove_dead_code()  argument
19534 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;  in opt_remove_dead_code()
19535 	int insn_cnt = env->prog->len;  in opt_remove_dead_code()
19547 		err = verifier_remove_insns(env, i, j);  in opt_remove_dead_code()
19550 		insn_cnt = env->prog->len;  in opt_remove_dead_code()
19558 static int opt_remove_nops(struct bpf_verifier_env *env)  in opt_remove_nops()  argument
19561 	struct bpf_insn *insn = env->prog->insnsi;  in opt_remove_nops()
19562 	int insn_cnt = env->prog->len;  in opt_remove_nops()
19569 		err = verifier_remove_insns(env, i, 1);  in opt_remove_nops()
19579 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,  in opt_subreg_zext_lo32_rnd_hi32()  argument
19583 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in opt_subreg_zext_lo32_rnd_hi32()
19584 	int i, patch_len, delta = 0, len = env->prog->len;  in opt_subreg_zext_lo32_rnd_hi32()
19585 	struct bpf_insn *insns = env->prog->insnsi;  in opt_subreg_zext_lo32_rnd_hi32()
19617 			if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {  in opt_subreg_zext_lo32_rnd_hi32()
19656 			verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");  in opt_subreg_zext_lo32_rnd_hi32()
19666 		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);  in opt_subreg_zext_lo32_rnd_hi32()
19669 		env->prog = new_prog;  in opt_subreg_zext_lo32_rnd_hi32()
19671 		aux = env->insn_aux_data;  in opt_subreg_zext_lo32_rnd_hi32()
19683 static int convert_ctx_accesses(struct bpf_verifier_env *env)  in convert_ctx_accesses()  argument
19685 	struct bpf_subprog_info *subprogs = env->subprog_info;  in convert_ctx_accesses()
19686 	const struct bpf_verifier_ops *ops = env->ops;  in convert_ctx_accesses()
19688 	const int insn_cnt = env->prog->len;  in convert_ctx_accesses()
19689 	struct bpf_insn *epilogue_buf = env->epilogue_buf;  in convert_ctx_accesses()
19690 	struct bpf_insn *insn_buf = env->insn_buf;  in convert_ctx_accesses()
19699 		epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,  in convert_ctx_accesses()
19702 			verbose(env, "bpf verifier is misconfigured\n");  in convert_ctx_accesses()
19710 			insn_buf[cnt++] = env->prog->insnsi[0];  in convert_ctx_accesses()
19711 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);  in convert_ctx_accesses()
19714 			env->prog = new_prog;  in convert_ctx_accesses()
19719 	if (ops->gen_prologue || env->seen_direct_write) {  in convert_ctx_accesses()
19721 			verbose(env, "bpf verifier is misconfigured\n");  in convert_ctx_accesses()
19724 		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,  in convert_ctx_accesses()
19725 					env->prog);  in convert_ctx_accesses()
19727 			verbose(env, "bpf verifier is misconfigured\n");  in convert_ctx_accesses()
19730 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);  in convert_ctx_accesses()
19734 			env->prog = new_prog;  in convert_ctx_accesses()
19740 		WARN_ON(adjust_jmp_off(env->prog, 0, delta));  in convert_ctx_accesses()
19742 	if (bpf_prog_is_offloaded(env->prog->aux))  in convert_ctx_accesses()
19745 	insn = env->prog->insnsi + delta;  in convert_ctx_accesses()
19770 			   env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {  in convert_ctx_accesses()
19772 			env->prog->aux->num_exentries++;  in convert_ctx_accesses()
19798 		    env->insn_aux_data[i + delta].sanitize_stack_spill) {  in convert_ctx_accesses()
19805 			new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);  in convert_ctx_accesses()
19810 			env->prog = new_prog;  in convert_ctx_accesses()
19815 		switch ((int)env->insn_aux_data[i + delta].ptr_type) {  in convert_ctx_accesses()
19847 				env->prog->aux->num_exentries++;  in convert_ctx_accesses()
19852 				verbose(env, "sign extending loads from arena are not supported yet\n");  in convert_ctx_accesses()
19856 			env->prog->aux->num_exentries++;  in convert_ctx_accesses()
19862 		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;  in convert_ctx_accesses()
19878 				verbose(env, "bpf verifier narrow ctx access misconfigured\n");  in convert_ctx_accesses()
19893 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,  in convert_ctx_accesses()
19897 			verbose(env, "bpf verifier is misconfigured\n");  in convert_ctx_accesses()
19905 				verbose(env, "bpf verifier narrow ctx load misconfigured\n");  in convert_ctx_accesses()
19930 		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in convert_ctx_accesses()
19937 		env->prog = new_prog;  in convert_ctx_accesses()
19944 static int jit_subprogs(struct bpf_verifier_env *env)  in jit_subprogs()  argument
19946 	struct bpf_prog *prog = env->prog, **func, *tmp;  in jit_subprogs()
19953 	if (env->subprog_cnt <= 1)  in jit_subprogs()
19964 		subprog = find_subprog(env, i + insn->imm + 1);  in jit_subprogs()
19977 		env->insn_aux_data[i].call_imm = insn->imm;  in jit_subprogs()
20000 	func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);  in jit_subprogs()
20004 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
20006 		subprog_end = env->subprog_info[i + 1].start;  in jit_subprogs()
20043 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;  in jit_subprogs()
20051 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;  in jit_subprogs()
20070 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;  in jit_subprogs()
20071 		func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;  in jit_subprogs()
20073 			func[i]->aux->exception_boundary = env->seen_exception;  in jit_subprogs()
20086 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
20113 		func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
20114 		func[i]->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
20116 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
20120 			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");  in jit_subprogs()
20131 	for (i = 1; i < env->subprog_cnt; i++) {  in jit_subprogs()
20137 	for (i = 1; i < env->subprog_cnt; i++)  in jit_subprogs()
20146 			insn[0].imm = env->insn_aux_data[i].call_imm;  in jit_subprogs()
20153 		insn->off = env->insn_aux_data[i].call_imm;  in jit_subprogs()
20154 		subprog = find_subprog(env, i + insn->off + 1);  in jit_subprogs()
20164 	prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;  in jit_subprogs()
20165 	prog->aux->real_func_cnt = env->subprog_cnt;  in jit_subprogs()
20166 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;  in jit_subprogs()
20183 	for (i = 0; i < env->subprog_cnt; i++) {  in jit_subprogs()
20198 		insn->imm = env->insn_aux_data[i].call_imm;  in jit_subprogs()
20204 static int fixup_call_args(struct bpf_verifier_env *env)  in fixup_call_args()  argument
20207 	struct bpf_prog *prog = env->prog;  in fixup_call_args()
20214 	if (env->prog->jit_requested &&  in fixup_call_args()
20215 	    !bpf_prog_is_offloaded(env->prog->aux)) {  in fixup_call_args()
20216 		err = jit_subprogs(env);  in fixup_call_args()
20224 		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");  in fixup_call_args()
20227 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {  in fixup_call_args()
20231 		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");  in fixup_call_args()
20239 			verbose(env, "callbacks are not allowed in non-JITed programs\n");  in fixup_call_args()
20245 		depth = get_callee_stack_depth(env, insn, i);  in fixup_call_args()
20256 static void specialize_kfunc(struct bpf_verifier_env *env,  in specialize_kfunc()  argument
20259 	struct bpf_prog *prog = env->prog;  in specialize_kfunc()
20277 		seen_direct_write = env->seen_direct_write;  in specialize_kfunc()
20278 		is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);  in specialize_kfunc()
20283 		/* restore env->seen_direct_write to its original value, since  in specialize_kfunc()
20286 		env->seen_direct_write = seen_direct_write;  in specialize_kfunc()
20307 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in fixup_kfunc_call()  argument
20313 		verbose(env, "invalid kernel function call not eliminated in verifier pass\n");  in fixup_kfunc_call()
20323 	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);  in fixup_kfunc_call()
20325 		verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",  in fixup_kfunc_call()
20336 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;  in fixup_kfunc_call()
20338 		u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;  in fixup_kfunc_call()
20341 			verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n",  in fixup_kfunc_call()
20354 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;  in fixup_kfunc_call()
20358 			verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n",  in fixup_kfunc_call()
20365 			verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",  in fixup_kfunc_call()
20377 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;  in fixup_kfunc_call()
20388 			verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",  in fixup_kfunc_call()
20393 		__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,  in fixup_kfunc_call()
20400 		struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) };  in fixup_kfunc_call()
20411 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)  in add_hidden_subprog()  argument
20413 	struct bpf_subprog_info *info = env->subprog_info;  in add_hidden_subprog()
20414 	int cnt = env->subprog_cnt;  in add_hidden_subprog()
20418 	if (env->hidden_subprog_cnt) {  in add_hidden_subprog()
20419 		verbose(env, "verifier internal error: only one hidden subprog supported\n");  in add_hidden_subprog()
20426 	prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);  in add_hidden_subprog()
20429 	env->prog = prog;  in add_hidden_subprog()
20432 	env->subprog_cnt++;  in add_hidden_subprog()
20433 	env->hidden_subprog_cnt++;  in add_hidden_subprog()
20440 static int do_misc_fixups(struct bpf_verifier_env *env)  in do_misc_fixups()  argument
20442 	struct bpf_prog *prog = env->prog;  in do_misc_fixups()
20450 	struct bpf_insn *insn_buf = env->insn_buf;  in do_misc_fixups()
20454 	struct bpf_subprog_info *subprogs = env->subprog_info;  in do_misc_fixups()
20458 	if (env->seen_exception && !env->exception_callback_subprog) {  in do_misc_fixups()
20460 			env->prog->insnsi[insn_cnt - 1],  in do_misc_fixups()
20465 		ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch));  in do_misc_fixups()
20468 		prog = env->prog;  in do_misc_fixups()
20471 		env->exception_callback_subprog = env->subprog_cnt - 1;  in do_misc_fixups()
20473 		mark_subprog_exc_cb(env, env->exception_callback_subprog);  in do_misc_fixups()
20479 			    (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {  in do_misc_fixups()
20489 		if (env->insn_aux_data[i + delta].needs_zext)  in do_misc_fixups()
20514 			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);  in do_misc_fixups()
20519 			env->prog = prog = new_prog;  in do_misc_fixups()
20609 			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);  in do_misc_fixups()
20614 			env->prog = prog = new_prog;  in do_misc_fixups()
20639 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20644 			env->prog = prog = new_prog;  in do_misc_fixups()
20653 			cnt = env->ops->gen_ld_abs(insn, insn_buf);  in do_misc_fixups()
20655 				verbose(env, "bpf verifier is misconfigured\n");  in do_misc_fixups()
20659 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20664 			env->prog = prog = new_prog;  in do_misc_fixups()
20678 			aux = &env->insn_aux_data[i + delta];  in do_misc_fixups()
20712 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20717 			env->prog = prog = new_prog;  in do_misc_fixups()
20735 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20740 			env->prog = prog = new_prog;  in do_misc_fixups()
20750 			ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);  in do_misc_fixups()
20756 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20761 			env->prog = prog = new_prog;  in do_misc_fixups()
20783 			if (!allow_tail_call_in_subprogs(env))  in do_misc_fixups()
20795 			aux = &env->insn_aux_data[i + delta];  in do_misc_fixups()
20796 			if (env->bpf_capable && !prog->blinding_requested &&  in do_misc_fixups()
20810 					verbose(env, "adding tail call poke descriptor failed\n");  in do_misc_fixups()
20828 				verbose(env, "tail_call abusing map_ptr\n");  in do_misc_fixups()
20841 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20846 			env->prog = prog = new_prog;  in do_misc_fixups()
20874 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20879 			env->prog = prog = new_prog;  in do_misc_fixups()
20885 			if (!in_sleepable(env) ||  in do_misc_fixups()
20886 			    env->insn_aux_data[i + delta].storage_get_func_atomic)  in do_misc_fixups()
20893 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20898 			env->prog = prog = new_prog;  in do_misc_fixups()
20904 		if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {  in do_misc_fixups()
20912 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
20917 			env->prog = prog = new_prog;  in do_misc_fixups()
20936 			aux = &env->insn_aux_data[i + delta];  in do_misc_fixups()
20948 					verbose(env, "bpf verifier is misconfigured\n");  in do_misc_fixups()
20952 				new_prog = bpf_patch_insn_data(env, i + delta,  in do_misc_fixups()
20958 				env->prog = prog = new_prog;  in do_misc_fixups()
21035 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,  in do_misc_fixups()
21041 			env->prog = prog = new_prog;  in do_misc_fixups()
21049 		    verifier_inlines_helper_call(env, insn->imm)) {  in do_misc_fixups()
21060 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
21065 			env->prog = prog = new_prog;  in do_misc_fixups()
21085 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
21090 			env->prog = prog = new_prog;  in do_misc_fixups()
21113 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
21118 			env->prog = prog = new_prog;  in do_misc_fixups()
21129 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);  in do_misc_fixups()
21133 			env->prog = prog = new_prog;  in do_misc_fixups()
21144 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);  in do_misc_fixups()
21148 			env->prog = prog = new_prog;  in do_misc_fixups()
21199 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
21204 			env->prog = prog = new_prog;  in do_misc_fixups()
21217 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);  in do_misc_fixups()
21222 			env->prog = prog = new_prog;  in do_misc_fixups()
21227 		fn = env->ops->get_func_proto(insn->imm, env->prog);  in do_misc_fixups()
21232 			verbose(env,  in do_misc_fixups()
21250 	env->prog->aux->stack_depth = subprogs[0].stack_depth;  in do_misc_fixups()
21251 	for (i = 0; i < env->subprog_cnt; i++) {  in do_misc_fixups()
21258 			verbose(env, "verifier bug: stack_slots supports may_goto only\n");  in do_misc_fixups()
21266 		insn_buf[1] = env->prog->insnsi[subprog_start];  in do_misc_fixups()
21268 		new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2);  in do_misc_fixups()
21271 		env->prog = prog = new_prog;  in do_misc_fixups()
21278 		WARN_ON(adjust_jmp_off(env->prog, subprog_start, 1));  in do_misc_fixups()
21287 			verbose(env, "bpf verifier is misconfigured\n");  in do_misc_fixups()
21293 			verbose(env, "tracking tail call prog failed\n");  in do_misc_fixups()
21298 	sort_kfunc_descs_by_imm_off(env->prog);  in do_misc_fixups()
21303 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,  in inline_bpf_loop()  argument
21316 	struct bpf_insn *insn_buf = env->insn_buf;  in inline_bpf_loop()
21365 	new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt);  in inline_bpf_loop()
21370 	callback_start = env->subprog_info[callback_subprogno].start;  in inline_bpf_loop()
21395 static int optimize_bpf_loop(struct bpf_verifier_env *env)  in optimize_bpf_loop()  argument
21397 	struct bpf_subprog_info *subprogs = env->subprog_info;  in optimize_bpf_loop()
21399 	struct bpf_insn *insn = env->prog->insnsi;  in optimize_bpf_loop()
21400 	int insn_cnt = env->prog->len;  in optimize_bpf_loop()
21407 			&env->insn_aux_data[i + delta].loop_inline_state;  in optimize_bpf_loop()
21413 			new_prog = inline_bpf_loop(env,  in optimize_bpf_loop()
21422 			env->prog  = new_prog;  in optimize_bpf_loop()
21435 	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;  in optimize_bpf_loop()
21443 static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)  in remove_fastcall_spills_fills()  argument
21445 	struct bpf_subprog_info *subprog = env->subprog_info;  in remove_fastcall_spills_fills()
21446 	struct bpf_insn_aux_data *aux = env->insn_aux_data;  in remove_fastcall_spills_fills()
21447 	struct bpf_insn *insn = env->prog->insnsi;  in remove_fastcall_spills_fills()
21448 	int insn_cnt = env->prog->len;  in remove_fastcall_spills_fills()
21474 static void free_states(struct bpf_verifier_env *env)  in free_states()  argument
21479 	sl = env->free_list;  in free_states()
21486 	env->free_list = NULL;  in free_states()
21488 	if (!env->explored_states)  in free_states()
21491 	for (i = 0; i < state_htab_size(env); i++) {  in free_states()
21492 		sl = env->explored_states[i];  in free_states()
21500 		env->explored_states[i] = NULL;  in free_states()
21504 static int do_check_common(struct bpf_verifier_env *env, int subprog)  in do_check_common()  argument
21506 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);  in do_check_common()
21507 	struct bpf_subprog_info *sub = subprog_info(env, subprog);  in do_check_common()
21512 	env->prev_linfo = NULL;  in do_check_common()
21513 	env->pass_cnt++;  in do_check_common()
21526 	env->cur_state = state;  in do_check_common()
21527 	init_func_state(env, state->frame[0],  in do_check_common()
21531 	state->first_insn_idx = env->subprog_info[subprog].start;  in do_check_common()
21535 	if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {  in do_check_common()
21536 		const char *sub_name = subprog_name(env, subprog);  in do_check_common()
21540 		verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);  in do_check_common()
21541 		ret = btf_prepare_func_args(env, subprog);  in do_check_common()
21545 		if (subprog_is_exc_cb(env, subprog)) {  in do_check_common()
21552 				verbose(env, "exception cb only supports single integer argument\n");  in do_check_common()
21563 				mark_reg_known_zero(env, regs, i);  in do_check_common()
21566 				mark_reg_unknown(env, regs, i);  in do_check_common()
21569 				__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);  in do_check_common()
21574 				mark_reg_known_zero(env, regs, i);  in do_check_common()
21576 				reg->id = ++env->id_gen;  in do_check_common()
21585 				mark_reg_known_zero(env, regs, i);  in do_check_common()
21588 				reg->id = ++env->id_gen;  in do_check_common()
21591 				mark_reg_unknown(env, regs, i);  in do_check_common()
21604 		if (env->prog->aux->func_info_aux) {  in do_check_common()
21605 			ret = btf_prepare_func_args(env, 0);  in do_check_common()
21607 				env->prog->aux->func_info_aux[0].unreliable = true;  in do_check_common()
21612 		mark_reg_known_zero(env, regs, BPF_REG_1);  in do_check_common()
21615 	ret = do_check(env);  in do_check_common()
21620 	if (env->cur_state) {  in do_check_common()
21621 		free_verifier_state(env->cur_state, true);  in do_check_common()
21622 		env->cur_state = NULL;  in do_check_common()
21624 	while (!pop_stack(env, NULL, NULL, false));  in do_check_common()
21626 		bpf_vlog_reset(&env->log, 0);  in do_check_common()
21627 	free_states(env);  in do_check_common()
21651 static int do_check_subprogs(struct bpf_verifier_env *env)  in do_check_subprogs()  argument
21653 	struct bpf_prog_aux *aux = env->prog->aux;  in do_check_subprogs()
21661 	if (env->exception_callback_subprog)  in do_check_subprogs()
21662 		subprog_aux(env, env->exception_callback_subprog)->called = true;  in do_check_subprogs()
21666 	for (i = 1; i < env->subprog_cnt; i++) {  in do_check_subprogs()
21667 		if (!subprog_is_global(env, i))  in do_check_subprogs()
21670 		sub_aux = subprog_aux(env, i);  in do_check_subprogs()
21674 		env->insn_idx = env->subprog_info[i].start;  in do_check_subprogs()
21675 		WARN_ON_ONCE(env->insn_idx == 0);  in do_check_subprogs()
21676 		ret = do_check_common(env, i);  in do_check_subprogs()
21679 		} else if (env->log.level & BPF_LOG_LEVEL) {  in do_check_subprogs()
21680 			verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",  in do_check_subprogs()
21681 				i, subprog_name(env, i));  in do_check_subprogs()
21701 static int do_check_main(struct bpf_verifier_env *env)  in do_check_main()  argument
21705 	env->insn_idx = 0;  in do_check_main()
21706 	ret = do_check_common(env, 0);  in do_check_main()
21708 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;  in do_check_main()
21713 static void print_verification_stats(struct bpf_verifier_env *env)  in print_verification_stats()  argument
21717 	if (env->log.level & BPF_LOG_STATS) {  in print_verification_stats()
21718 		verbose(env, "verification time %lld usec\n",  in print_verification_stats()
21719 			div_u64(env->verification_time, 1000));  in print_verification_stats()
21720 		verbose(env, "stack depth ");  in print_verification_stats()
21721 		for (i = 0; i < env->subprog_cnt; i++) {  in print_verification_stats()
21722 			u32 depth = env->subprog_info[i].stack_depth;  in print_verification_stats()
21724 			verbose(env, "%d", depth);  in print_verification_stats()
21725 			if (i + 1 < env->subprog_cnt)  in print_verification_stats()
21726 				verbose(env, "+");  in print_verification_stats()
21728 		verbose(env, "\n");  in print_verification_stats()
21730 	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "  in print_verification_stats()
21732 		env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,  in print_verification_stats()
21733 		env->max_states_per_insn, env->total_states,  in print_verification_stats()
21734 		env->peak_states, env->longest_mark_read_walk);  in print_verification_stats()
21737 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)  in check_struct_ops_btf_id()  argument
21743 	struct bpf_prog *prog = env->prog;  in check_struct_ops_btf_id()
21750 		verbose(env, "struct ops programs must have a GPL compatible license\n");  in check_struct_ops_btf_id()
21759 		/* Make sure st_ops is valid through the lifetime of env */  in check_struct_ops_btf_id()
21760 		env->attach_btf_mod = btf_try_get_module(btf);  in check_struct_ops_btf_id()
21761 		if (!env->attach_btf_mod) {  in check_struct_ops_btf_id()
21762 			verbose(env, "struct_ops module %s is not found\n",  in check_struct_ops_btf_id()
21771 		verbose(env, "attach_btf_id %u is not a supported struct\n",  in check_struct_ops_btf_id()
21780 		verbose(env, "attach to invalid member idx %u of struct %s\n",  in check_struct_ops_btf_id()
21790 		verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",  in check_struct_ops_btf_id()
21797 		verbose(env, "attach to unsupported member %s of struct %s\n",  in check_struct_ops_btf_id()
21806 			verbose(env, "attach to unsupported member %s of struct %s\n",  in check_struct_ops_btf_id()
21820 	env->ops = st_ops->verifier_ops;  in check_struct_ops_btf_id()
22205 static int check_attach_btf_id(struct bpf_verifier_env *env)  in check_attach_btf_id()  argument
22207 	struct bpf_prog *prog = env->prog;  in check_attach_btf_id()
22219 		verbose(env, "Syscall programs can only be sleepable\n");  in check_attach_btf_id()
22224 …verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepa…  in check_attach_btf_id()
22229 		return check_struct_ops_btf_id(env);  in check_attach_btf_id()
22236 	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);  in check_attach_btf_id()
22242 		 * inherit env->ops and expected_attach_type for the rest of the  in check_attach_btf_id()
22245 		env->ops = bpf_verifier_ops[tgt_prog->type];  in check_attach_btf_id()
22269 		ret = bpf_lsm_verify_prog(&env->log, prog);  in check_attach_btf_id()
22303 	struct bpf_verifier_env *env;  in bpf_check()  local
22315 	env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);  in bpf_check()
22316 	if (!env)  in bpf_check()
22319 	env->bt.env = env;  in bpf_check()
22322 	env->insn_aux_data =  in bpf_check()
22325 	if (!env->insn_aux_data)  in bpf_check()
22328 		env->insn_aux_data[i].orig_idx = i;  in bpf_check()
22329 	env->prog = *prog;  in bpf_check()
22330 	env->ops = bpf_verifier_ops[env->prog->type];  in bpf_check()
22331 	env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);  in bpf_check()
22333 	env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);  in bpf_check()
22334 	env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);  in bpf_check()
22335 	env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);  in bpf_check()
22336 	env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);  in bpf_check()
22337 	env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);  in bpf_check()
22348 	ret = bpf_vlog_init(&env->log, attr->log_level,  in bpf_check()
22354 	mark_verifier_state_clean(env);  in bpf_check()
22358 		verbose(env, "in-kernel BTF is malformed\n");  in bpf_check()
22363 	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);  in bpf_check()
22365 		env->strict_alignment = true;  in bpf_check()
22367 		env->strict_alignment = false;  in bpf_check()
22370 		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;  in bpf_check()
22371 	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;  in bpf_check()
22373 	env->explored_states = kvcalloc(state_htab_size(env),  in bpf_check()
22377 	if (!env->explored_states)  in bpf_check()
22380 	ret = check_btf_info_early(env, attr, uattr);  in bpf_check()
22384 	ret = add_subprog_and_kfunc(env);  in bpf_check()
22388 	ret = check_subprogs(env);  in bpf_check()
22392 	ret = check_btf_info(env, attr, uattr);  in bpf_check()
22396 	ret = check_attach_btf_id(env);  in bpf_check()
22400 	ret = resolve_pseudo_ldimm64(env);  in bpf_check()
22404 	if (bpf_prog_is_offloaded(env->prog->aux)) {  in bpf_check()
22405 		ret = bpf_prog_offload_verifier_prep(env->prog);  in bpf_check()
22410 	ret = check_cfg(env);  in bpf_check()
22414 	ret = mark_fastcall_patterns(env);  in bpf_check()
22418 	ret = do_check_main(env);  in bpf_check()
22419 	ret = ret ?: do_check_subprogs(env);  in bpf_check()
22421 	if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))  in bpf_check()
22422 		ret = bpf_prog_offload_finalize(env);  in bpf_check()
22425 	kvfree(env->explored_states);  in bpf_check()
22431 		ret = remove_fastcall_spills_fills(env);  in bpf_check()
22434 		ret = check_max_stack_depth(env);  in bpf_check()
22438 		ret = optimize_bpf_loop(env);  in bpf_check()
22442 			opt_hard_wire_dead_code_branches(env);  in bpf_check()
22444 			ret = opt_remove_dead_code(env);  in bpf_check()
22446 			ret = opt_remove_nops(env);  in bpf_check()
22449 			sanitize_dead_code(env);  in bpf_check()
22454 		ret = convert_ctx_accesses(env);  in bpf_check()
22457 		ret = do_misc_fixups(env);  in bpf_check()
22462 	if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {  in bpf_check()
22463 		ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);  in bpf_check()
22464 		env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret  in bpf_check()
22469 		ret = fixup_call_args(env);  in bpf_check()
22471 	env->verification_time = ktime_get_ns() - start_time;  in bpf_check()
22472 	print_verification_stats(env);  in bpf_check()
22473 	env->prog->aux->verified_insns = env->insn_processed;  in bpf_check()
22476 	err = bpf_vlog_finalize(&env->log, &log_true_size);  in bpf_check()
22490 	if (env->used_map_cnt) {  in bpf_check()
22492 		env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,  in bpf_check()
22493 							  sizeof(env->used_maps[0]),  in bpf_check()
22496 		if (!env->prog->aux->used_maps) {  in bpf_check()
22501 		memcpy(env->prog->aux->used_maps, env->used_maps,  in bpf_check()
22502 		       sizeof(env->used_maps[0]) * env->used_map_cnt);  in bpf_check()
22503 		env->prog->aux->used_map_cnt = env->used_map_cnt;  in bpf_check()
22505 	if (env->used_btf_cnt) {  in bpf_check()
22507 		env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,  in bpf_check()
22508 							  sizeof(env->used_btfs[0]),  in bpf_check()
22510 		if (!env->prog->aux->used_btfs) {  in bpf_check()
22515 		memcpy(env->prog->aux->used_btfs, env->used_btfs,  in bpf_check()
22516 		       sizeof(env->used_btfs[0]) * env->used_btf_cnt);  in bpf_check()
22517 		env->prog->aux->used_btf_cnt = env->used_btf_cnt;  in bpf_check()
22519 	if (env->used_map_cnt || env->used_btf_cnt) {  in bpf_check()
22523 		convert_pseudo_ld_imm64(env);  in bpf_check()
22526 	adjust_btf_func(env);  in bpf_check()
22529 	if (!env->prog->aux->used_maps)  in bpf_check()
22533 		release_maps(env);  in bpf_check()
22534 	if (!env->prog->aux->used_btfs)  in bpf_check()
22535 		release_btfs(env);  in bpf_check()
22540 	if (env->prog->type == BPF_PROG_TYPE_EXT)  in bpf_check()
22541 		env->prog->expected_attach_type = 0;  in bpf_check()
22543 	*prog = env->prog;  in bpf_check()
22545 	module_put(env->attach_btf_mod);  in bpf_check()
22549 	vfree(env->insn_aux_data);  in bpf_check()
22551 	kvfree(env);  in bpf_check()