107 bool bpf_prog_has_trampoline(const struct bpf_prog *prog)  in bpf_prog_has_trampoline()  argument
109 	enum bpf_attach_type eatype = prog->expected_attach_type;  in bpf_prog_has_trampoline()
110 	enum bpf_prog_type ptype = prog->type;  in bpf_prog_has_trampoline()
247 			*ip_arg |= link->link.prog->call_get_func_ip;  in bpf_trampoline_get_progs()
504 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)  in bpf_attach_type_to_tramp()  argument
506 	switch (prog->expected_attach_type) {  in bpf_attach_type_to_tramp()
514 		if (!prog->aux->attach_func_proto->type)  in bpf_attach_type_to_tramp()
533 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_link_prog()
547 		tr->extension_prog = link->link.prog;  in __bpf_trampoline_link_prog()
549 					  link->link.prog->bpf_func);  in __bpf_trampoline_link_prog()
557 		if (link_exiting->link.prog != link->link.prog)  in __bpf_trampoline_link_prog()
588 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_unlink_prog()
639 static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,  in cgroup_shim_alloc()  argument
660 	p->aux->attach_func_proto = prog->aux->attach_func_proto;  in cgroup_shim_alloc()
661 	p->aux->attach_btf_id = prog->aux->attach_btf_id;  in cgroup_shim_alloc()
662 	p->aux->attach_btf = prog->aux->attach_btf;  in cgroup_shim_alloc()
682 			struct bpf_prog *p = link->link.prog;  in cgroup_shim_find()
692 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,  in bpf_trampoline_link_cgroup_shim()  argument
702 	err = bpf_check_attach_target(NULL, prog, NULL,  in bpf_trampoline_link_cgroup_shim()
703 				      prog->aux->attach_btf_id,  in bpf_trampoline_link_cgroup_shim()
708 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_link_cgroup_shim()
709 					 prog->aux->attach_btf_id);  in bpf_trampoline_link_cgroup_shim()
711 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_link_cgroup_shim()
730 	shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);  in bpf_trampoline_link_cgroup_shim()
758 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)  in bpf_trampoline_unlink_cgroup_shim()  argument
765 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_unlink_cgroup_shim()
766 					 prog->aux->attach_btf_id);  in bpf_trampoline_unlink_cgroup_shim()
768 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_unlink_cgroup_shim()
861 static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_enter_recur()  argument
869 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_recur()
870 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_recur()
876 static void notrace update_prog_stats(struct bpf_prog *prog,  in update_prog_stats()  argument
891 		stats = this_cpu_ptr(prog->stats);  in update_prog_stats()
899 static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_recur()  argument
905 	update_prog_stats(prog, start);  in __bpf_prog_exit_recur()
906 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_recur()
911 static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,  in __bpf_prog_enter_lsm_cgroup()  argument
926 static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_lsm_cgroup()  argument
936 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,  in __bpf_prog_enter_sleepable_recur()  argument
945 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_sleepable_recur()
946 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_sleepable_recur()
952 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable_recur()  argument
957 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable_recur()
958 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_sleepable_recur()
963 static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,  in __bpf_prog_enter_sleepable()  argument
975 static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable()  argument
980 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable()
985 static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,  in __bpf_prog_enter()  argument
997 static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit()  argument
1003 	update_prog_stats(prog, start);  in __bpf_prog_exit()
1018 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog)  in bpf_trampoline_enter()  argument
1020 	bool sleepable = prog->sleepable;  in bpf_trampoline_enter()
1022 	if (bpf_prog_check_recur(prog))  in bpf_trampoline_enter()
1026 	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&  in bpf_trampoline_enter()
1027 	    prog->expected_attach_type == BPF_LSM_CGROUP)  in bpf_trampoline_enter()
1033 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog)  in bpf_trampoline_exit()  argument
1035 	bool sleepable = prog->sleepable;  in bpf_trampoline_exit()
1037 	if (bpf_prog_check_recur(prog))  in bpf_trampoline_exit()
1041 	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&  in bpf_trampoline_exit()
1042 	    prog->expected_attach_type == BPF_LSM_CGROUP)  in bpf_trampoline_exit()