Lines Matching refs:prog
75 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
748 void bpf_program__unload(struct bpf_program *prog) in bpf_program__unload() argument
750 if (!prog) in bpf_program__unload()
753 zclose(prog->fd); in bpf_program__unload()
755 zfree(&prog->func_info); in bpf_program__unload()
756 zfree(&prog->line_info); in bpf_program__unload()
759 static void bpf_program__exit(struct bpf_program *prog) in bpf_program__exit() argument
761 if (!prog) in bpf_program__exit()
764 bpf_program__unload(prog); in bpf_program__exit()
765 zfree(&prog->name); in bpf_program__exit()
766 zfree(&prog->sec_name); in bpf_program__exit()
767 zfree(&prog->insns); in bpf_program__exit()
768 zfree(&prog->reloc_desc); in bpf_program__exit()
770 prog->nr_reloc = 0; in bpf_program__exit()
771 prog->insns_cnt = 0; in bpf_program__exit()
772 prog->sec_idx = -1; in bpf_program__exit()
796 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, in bpf_object__init_prog() argument
806 memset(prog, 0, sizeof(*prog)); in bpf_object__init_prog()
807 prog->obj = obj; in bpf_object__init_prog()
809 prog->sec_idx = sec_idx; in bpf_object__init_prog()
810 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
811 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
813 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
815 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
816 prog->fd = -1; in bpf_object__init_prog()
817 prog->exception_cb_idx = -1; in bpf_object__init_prog()
824 prog->autoload = false; in bpf_object__init_prog()
828 prog->autoload = true; in bpf_object__init_prog()
831 prog->autoattach = true; in bpf_object__init_prog()
834 prog->log_level = obj->log_level; in bpf_object__init_prog()
836 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
837 if (!prog->sec_name) in bpf_object__init_prog()
840 prog->name = strdup(name); in bpf_object__init_prog()
841 if (!prog->name) in bpf_object__init_prog()
844 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
845 if (!prog->insns) in bpf_object__init_prog()
847 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
852 bpf_program__exit(prog); in bpf_object__init_prog()
861 struct bpf_program *prog, *progs; in bpf_object__add_programs() local
917 prog = &progs[nr_progs]; in bpf_object__add_programs()
919 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, in bpf_object__add_programs()
925 prog->sym_global = true; in bpf_object__add_programs()
932 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
934 prog->mark_btf_static = true; in bpf_object__add_programs()
1052 const struct bpf_program *prog) in is_valid_st_ops_program() argument
1057 if (&obj->programs[i] == prog) in is_valid_st_ops_program()
1058 return prog->type == BPF_PROG_TYPE_STRUCT_OPS; in is_valid_st_ops_program()
1071 struct bpf_program *prog, *slot_prog; in bpf_object_adjust_struct_ops_autoload() local
1079 prog = &obj->programs[i]; in bpf_object_adjust_struct_ops_autoload()
1080 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) in bpf_object_adjust_struct_ops_autoload()
1094 if (prog != slot_prog) in bpf_object_adjust_struct_ops_autoload()
1103 prog->autoload = should_load; in bpf_object_adjust_struct_ops_autoload()
1156 struct bpf_program *prog; in bpf_map__init_kern_struct_ops() local
1221 prog = *(void **)mdata; in bpf_map__init_kern_struct_ops()
1226 if (st_ops->progs[i] && st_ops->progs[i] != prog) in bpf_map__init_kern_struct_ops()
1230 st_ops->progs[i] = prog; in bpf_map__init_kern_struct_ops()
1231 if (!prog) in bpf_map__init_kern_struct_ops()
1234 if (!is_valid_st_ops_program(obj, prog)) { in bpf_map__init_kern_struct_ops()
1255 prog->attach_btf_obj_fd = mod_btf->fd; in bpf_map__init_kern_struct_ops()
1260 if (!prog->attach_btf_id) { in bpf_map__init_kern_struct_ops()
1261 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1262 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1270 if (prog->attach_btf_id != kern_type_id) { in bpf_map__init_kern_struct_ops()
1272 map->name, mname, prog->name, prog->sec_name, prog->type, in bpf_map__init_kern_struct_ops()
1273 prog->attach_btf_id, kern_type_id); in bpf_map__init_kern_struct_ops()
1276 if (prog->expected_attach_type != kern_member_idx) { in bpf_map__init_kern_struct_ops()
1278 map->name, mname, prog->name, prog->sec_name, prog->type, in bpf_map__init_kern_struct_ops()
1279 prog->expected_attach_type, kern_member_idx); in bpf_map__init_kern_struct_ops()
1286 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
3411 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) in prog_needs_vmlinux_btf() argument
3413 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || in prog_needs_vmlinux_btf()
3414 prog->type == BPF_PROG_TYPE_LSM) in prog_needs_vmlinux_btf()
3420 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) in prog_needs_vmlinux_btf()
3433 struct bpf_program *prog; in obj_needs_vmlinux_btf() local
3452 bpf_object__for_each_program(prog, obj) { in obj_needs_vmlinux_btf()
3453 if (!prog->autoload) in obj_needs_vmlinux_btf()
3455 if (prog_needs_vmlinux_btf(prog)) in obj_needs_vmlinux_btf()
3515 struct bpf_program *prog = &obj->programs[i]; in bpf_object__sanitize_and_load_btf() local
3520 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) in bpf_object__sanitize_and_load_btf()
3530 if (strcmp(name, prog->name) != 0) in bpf_object__sanitize_and_load_btf()
4390 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) in prog_is_subprog() argument
4392 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; in prog_is_subprog()
4399 struct bpf_program *prog; in bpf_object__find_program_by_name() local
4401 bpf_object__for_each_program(prog, obj) { in bpf_object__find_program_by_name()
4402 if (prog_is_subprog(obj, prog)) in bpf_object__find_program_by_name()
4404 if (!strcmp(prog->name, name)) in bpf_object__find_program_by_name()
4405 return prog; in bpf_object__find_program_by_name()
4447 static int bpf_program__record_reloc(struct bpf_program *prog, in bpf_program__record_reloc() argument
4452 struct bpf_insn *insn = &prog->insns[insn_idx]; in bpf_program__record_reloc()
4453 size_t map_idx, nr_maps = prog->obj->nr_maps; in bpf_program__record_reloc()
4454 struct bpf_object *obj = prog->obj; in bpf_program__record_reloc()
4462 prog->name, sym_name, insn_idx, insn->code); in bpf_program__record_reloc()
4478 prog->name, sym_name, sym_idx); in bpf_program__record_reloc()
4482 prog->name, i, ext->name, ext->sym_idx, insn_idx); in bpf_program__record_reloc()
4495 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); in bpf_program__record_reloc()
4502 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4507 prog->name, sym_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4518 prog->name, sym_name, shdr_idx); in bpf_program__record_reloc()
4529 prog->name, sym_name, (size_t)sym->st_value, insn->imm); in bpf_program__record_reloc()
4555 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4565 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4571 prog->name, sym_sec_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4584 prog->name, sym_sec_name); in bpf_program__record_reloc()
4592 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4598 prog->name, sym_sec_name); in bpf_program__record_reloc()
4609 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) in prog_contains_insn() argument
4611 return insn_idx >= prog->sec_insn_off && in prog_contains_insn()
4612 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; in prog_contains_insn()
4619 struct bpf_program *prog; in find_prog_by_sec_insn() local
4626 prog = &obj->programs[m]; in find_prog_by_sec_insn()
4628 if (prog->sec_idx < sec_idx || in find_prog_by_sec_insn()
4629 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) in find_prog_by_sec_insn()
4637 prog = &obj->programs[l]; in find_prog_by_sec_insn()
4638 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) in find_prog_by_sec_insn()
4639 return prog; in find_prog_by_sec_insn()
4648 struct bpf_program *prog; in bpf_object__collect_prog_relos() local
4718 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); in bpf_object__collect_prog_relos()
4719 if (!prog) { in bpf_object__collect_prog_relos()
4725 relos = libbpf_reallocarray(prog->reloc_desc, in bpf_object__collect_prog_relos()
4726 prog->nr_reloc + 1, sizeof(*relos)); in bpf_object__collect_prog_relos()
4729 prog->reloc_desc = relos; in bpf_object__collect_prog_relos()
4732 insn_idx -= prog->sec_insn_off; in bpf_object__collect_prog_relos()
4733 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], in bpf_object__collect_prog_relos()
4738 prog->nr_reloc++; in bpf_object__collect_prog_relos()
5775 static int record_relo_core(struct bpf_program *prog, in record_relo_core() argument
5780 relos = libbpf_reallocarray(prog->reloc_desc, in record_relo_core()
5781 prog->nr_reloc + 1, sizeof(*relos)); in record_relo_core()
5784 relo = &relos[prog->nr_reloc]; in record_relo_core()
5788 prog->reloc_desc = relos; in record_relo_core()
5789 prog->nr_reloc++; in record_relo_core()
5793 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) in find_relo_core() argument
5798 for (i = 0; i < prog->nr_reloc; i++) { in find_relo_core()
5799 relo = &prog->reloc_desc[i]; in find_relo_core()
5809 static int bpf_core_resolve_relo(struct bpf_program *prog, in bpf_core_resolve_relo() argument
5818 const char *prog_name = prog->name; in bpf_core_resolve_relo()
5834 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); in bpf_core_resolve_relo()
5861 struct bpf_program *prog; in bpf_object__relocate_core() local
5902 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); in bpf_object__relocate_core()
5903 if (!prog) { in bpf_object__relocate_core()
5919 if (!prog->autoload) in bpf_object__relocate_core()
5926 insn_idx = insn_idx - prog->sec_insn_off; in bpf_object__relocate_core()
5927 if (insn_idx >= prog->insns_cnt) in bpf_object__relocate_core()
5929 insn = &prog->insns[insn_idx]; in bpf_object__relocate_core()
5931 err = record_relo_core(prog, rec, insn_idx); in bpf_object__relocate_core()
5934 prog->name, i, err); in bpf_object__relocate_core()
5938 if (prog->obj->gen_loader) in bpf_object__relocate_core()
5941 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); in bpf_object__relocate_core()
5944 prog->name, i, err); in bpf_object__relocate_core()
5948 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); in bpf_object__relocate_core()
5951 prog->name, i, insn_idx, err); in bpf_object__relocate_core()
5975 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, in poison_map_ldimm64() argument
5982 prog->name, relo_idx, insn_idx, map_idx, map->name); in poison_map_ldimm64()
6005 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx, in poison_kfunc_call() argument
6010 prog->name, relo_idx, insn_idx, ext->name); in poison_kfunc_call()
6031 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__relocate_data() argument
6035 for (i = 0; i < prog->nr_reloc; i++) { in bpf_object__relocate_data()
6036 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_object__relocate_data()
6037 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate_data()
6051 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6065 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6098 poison_kfunc_call(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6105 prog->name, i); in bpf_object__relocate_data()
6118 prog->name, i, relo->type); in bpf_object__relocate_data()
6127 const struct bpf_program *prog, in adjust_prog_btf_ext_info() argument
6142 if (prog->sec_idx != sec_idx) in adjust_prog_btf_ext_info()
6148 if (insn_off < prog->sec_insn_off) in adjust_prog_btf_ext_info()
6150 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) in adjust_prog_btf_ext_info()
6179 off_adj = prog->sub_insn_off - prog->sec_insn_off; in adjust_prog_btf_ext_info()
6197 const struct bpf_program *prog) in reloc_prog_func_and_line_info() argument
6210 if (main_prog != prog && !main_prog->func_info) in reloc_prog_func_and_line_info()
6213 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, in reloc_prog_func_and_line_info()
6220 prog->name, err); in reloc_prog_func_and_line_info()
6228 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); in reloc_prog_func_and_line_info()
6233 prog->name); in reloc_prog_func_and_line_info()
6238 if (main_prog != prog && !main_prog->line_info) in reloc_prog_func_and_line_info()
6241 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, in reloc_prog_func_and_line_info()
6248 prog->name, err); in reloc_prog_func_and_line_info()
6256 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); in reloc_prog_func_and_line_info()
6261 prog->name); in reloc_prog_func_and_line_info()
6276 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) in find_prog_insn_relo() argument
6278 if (!prog->nr_reloc) in find_prog_insn_relo()
6280 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, in find_prog_insn_relo()
6281 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); in find_prog_insn_relo()
6347 struct bpf_program *prog) in bpf_object__reloc_code() argument
6355 err = reloc_prog_func_and_line_info(obj, main_prog, prog); in bpf_object__reloc_code()
6359 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { in bpf_object__reloc_code()
6360 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6364 relo = find_prog_insn_relo(prog, insn_idx); in bpf_object__reloc_code()
6372 prog->name, insn_idx, relo->type); in bpf_object__reloc_code()
6396 prog->name, insn_idx); in bpf_object__reloc_code()
6405 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; in bpf_object__reloc_code()
6412 prog->name); in bpf_object__reloc_code()
6438 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6445 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; in bpf_object__reloc_code()
6448 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); in bpf_object__reloc_code()
6536 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__relocate_calls() argument
6552 err = bpf_object__reloc_code(obj, prog, prog); in bpf_object__relocate_calls()
6562 struct bpf_program *prog; in bpf_object__free_relocs() local
6567 prog = &obj->programs[i]; in bpf_object__free_relocs()
6568 zfree(&prog->reloc_desc); in bpf_object__free_relocs()
6569 prog->nr_reloc = 0; in bpf_object__free_relocs()
6602 static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog) in bpf_prog_assign_exc_cb() argument
6627 prog->name); in bpf_prog_assign_exc_cb()
6630 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0) in bpf_prog_assign_exc_cb()
6636 if (prog->exception_cb_idx >= 0) { in bpf_prog_assign_exc_cb()
6637 prog->exception_cb_idx = -1; in bpf_prog_assign_exc_cb()
6644 prog->name); in bpf_prog_assign_exc_cb()
6661 prog->name, subprog->name); in bpf_prog_assign_exc_cb()
6665 if (prog->exception_cb_idx >= 0) { in bpf_prog_assign_exc_cb()
6667 prog->name, subprog->name); in bpf_prog_assign_exc_cb()
6670 prog->exception_cb_idx = j; in bpf_prog_assign_exc_cb()
6674 if (prog->exception_cb_idx >= 0) in bpf_prog_assign_exc_cb()
6677 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); in bpf_prog_assign_exc_cb()
6724 static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog, in need_func_arg_type_fixup() argument
6743 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) { in need_func_arg_type_fixup()
6762 switch (prog->type) { in need_func_arg_type_fixup()
6791 prog->name, subprog_name, arg_idx, ctx_name); in need_func_arg_type_fixup()
6795 static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog) in clone_func_btf_info() argument
6847 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id); in clone_func_btf_info()
6864 static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog) in bpf_program_fixup_func_info() argument
6877 if (!obj->btf_ext || !prog->func_info) in bpf_program_fixup_func_info()
6888 if (global_ctx_map[i].prog_type != prog->type) in bpf_program_fixup_func_info()
6897 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids)); in bpf_program_fixup_func_info()
6900 for (i = 0; i < prog->func_info_cnt; i++) { in bpf_program_fixup_func_info()
6901 func_rec = prog->func_info + prog->func_info_rec_size * i; in bpf_program_fixup_func_info()
6930 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) { in bpf_program_fixup_func_info()
6932 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx; in bpf_program_fixup_func_info()
6949 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name)) in bpf_program_fixup_func_info()
6956 fn_id = clone_func_btf_info(btf, orig_fn_id, prog); in bpf_program_fixup_func_info()
7005 struct bpf_program *prog; in bpf_object__relocate() local
7027 prog = &obj->programs[i]; in bpf_object__relocate()
7028 for (j = 0; j < prog->nr_reloc; j++) { in bpf_object__relocate()
7029 struct reloc_desc *relo = &prog->reloc_desc[j]; in bpf_object__relocate()
7030 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate()
7046 prog = &obj->programs[i]; in bpf_object__relocate()
7050 if (prog_is_subprog(obj, prog)) in bpf_object__relocate()
7052 if (!prog->autoload) in bpf_object__relocate()
7055 err = bpf_object__relocate_calls(obj, prog); in bpf_object__relocate()
7058 prog->name, err); in bpf_object__relocate()
7062 err = bpf_prog_assign_exc_cb(obj, prog); in bpf_object__relocate()
7066 if (prog->exception_cb_idx >= 0) { in bpf_object__relocate()
7067 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; in bpf_object__relocate()
7075 err = bpf_object__append_subprog_code(obj, prog, subprog); in bpf_object__relocate()
7078 err = bpf_object__reloc_code(obj, prog, subprog); in bpf_object__relocate()
7085 prog = &obj->programs[i]; in bpf_object__relocate()
7086 if (prog_is_subprog(obj, prog)) in bpf_object__relocate()
7088 if (!prog->autoload) in bpf_object__relocate()
7092 err = bpf_object__relocate_data(obj, prog); in bpf_object__relocate()
7095 prog->name, err); in bpf_object__relocate()
7100 err = bpf_program_fixup_func_info(obj, prog); in bpf_object__relocate()
7103 prog->name, err); in bpf_object__relocate()
7300 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__sanitize_prog() argument
7302 struct bpf_insn *insn = prog->insns; in bpf_object__sanitize_prog()
7309 for (i = 0; i < prog->insns_cnt; i++, insn++) { in bpf_object__sanitize_prog()
7335 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
7339 static int libbpf_prepare_prog_load(struct bpf_program *prog, in libbpf_prepare_prog_load() argument
7345 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) in libbpf_prepare_prog_load()
7351 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) in libbpf_prepare_prog_load()
7355 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) in libbpf_prepare_prog_load()
7356 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; in libbpf_prepare_prog_load()
7358 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { in libbpf_prepare_prog_load()
7362 attach_name = strchr(prog->sec_name, '/'); in libbpf_prepare_prog_load()
7374 prog->name); in libbpf_prepare_prog_load()
7379 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); in libbpf_prepare_prog_load()
7384 prog->attach_btf_obj_fd = btf_obj_fd; in libbpf_prepare_prog_load()
7385 prog->attach_btf_id = btf_type_id; in libbpf_prepare_prog_load()
7398 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
7400 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, in bpf_object_load_prog() argument
7410 __u32 log_level = prog->log_level; in bpf_object_load_prog()
7416 switch (prog->type) { in bpf_object_load_prog()
7423 prog->name, prog->sec_name); in bpf_object_load_prog()
7426 if (prog->attach_btf_id == 0) { in bpf_object_load_prog()
7428 prog->name); in bpf_object_load_prog()
7440 prog_name = prog->name; in bpf_object_load_prog()
7441 load_attr.attach_prog_fd = prog->attach_prog_fd; in bpf_object_load_prog()
7442 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; in bpf_object_load_prog()
7443 load_attr.attach_btf_id = prog->attach_btf_id; in bpf_object_load_prog()
7445 load_attr.prog_ifindex = prog->prog_ifindex; in bpf_object_load_prog()
7450 load_attr.func_info = prog->func_info; in bpf_object_load_prog()
7451 load_attr.func_info_rec_size = prog->func_info_rec_size; in bpf_object_load_prog()
7452 load_attr.func_info_cnt = prog->func_info_cnt; in bpf_object_load_prog()
7453 load_attr.line_info = prog->line_info; in bpf_object_load_prog()
7454 load_attr.line_info_rec_size = prog->line_info_rec_size; in bpf_object_load_prog()
7455 load_attr.line_info_cnt = prog->line_info_cnt; in bpf_object_load_prog()
7458 load_attr.prog_flags = prog->prog_flags; in bpf_object_load_prog()
7466 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { in bpf_object_load_prog()
7467 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); in bpf_object_load_prog()
7470 prog->name, err); in bpf_object_load_prog()
7473 insns = prog->insns; in bpf_object_load_prog()
7474 insns_cnt = prog->insns_cnt; in bpf_object_load_prog()
7478 load_attr.expected_attach_type = prog->expected_attach_type; in bpf_object_load_prog()
7481 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, in bpf_object_load_prog()
7483 prog - obj->programs); in bpf_object_load_prog()
7495 if (prog->log_buf) { in bpf_object_load_prog()
7496 log_buf = prog->log_buf; in bpf_object_load_prog()
7497 log_buf_size = prog->log_size; in bpf_object_load_prog()
7520 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); in bpf_object_load_prog()
7524 prog->name, log_buf); in bpf_object_load_prog()
7532 map = &prog->obj->maps[i]; in bpf_object_load_prog()
7539 prog->name, map->real_name, cp); in bpf_object_load_prog()
7567 fixup_verifier_log(prog, log_buf, log_buf_size); in bpf_object_load_prog()
7570 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); in bpf_object_load_prog()
7575 prog->name, log_buf); in bpf_object_load_prog()
7635 static void fixup_log_failed_core_relo(struct bpf_program *prog, in fixup_log_failed_core_relo() argument
7657 relo = find_relo_core(prog, insn_idx); in fixup_log_failed_core_relo()
7661 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); in fixup_log_failed_core_relo()
7674 static void fixup_log_missing_map_load(struct bpf_program *prog, in fixup_log_missing_map_load() argument
7686 struct bpf_object *obj = prog->obj; in fixup_log_missing_map_load()
7707 static void fixup_log_missing_kfunc_call(struct bpf_program *prog, in fixup_log_missing_kfunc_call() argument
7719 struct bpf_object *obj = prog->obj; in fixup_log_missing_kfunc_call()
7740 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) in fixup_verifier_log() argument
7765 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, in fixup_verifier_log()
7774 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, in fixup_verifier_log()
7783 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz, in fixup_verifier_log()
7790 static int bpf_program_record_relos(struct bpf_program *prog) in bpf_program_record_relos() argument
7792 struct bpf_object *obj = prog->obj; in bpf_program_record_relos()
7795 for (i = 0; i < prog->nr_reloc; i++) { in bpf_program_record_relos()
7796 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_program_record_relos()
7836 struct bpf_program *prog; in bpf_object__load_progs() local
7841 prog = &obj->programs[i]; in bpf_object__load_progs()
7842 err = bpf_object__sanitize_prog(obj, prog); in bpf_object__load_progs()
7848 prog = &obj->programs[i]; in bpf_object__load_progs()
7849 if (prog_is_subprog(obj, prog)) in bpf_object__load_progs()
7851 if (!prog->autoload) { in bpf_object__load_progs()
7852 pr_debug("prog '%s': skipped loading\n", prog->name); in bpf_object__load_progs()
7855 prog->log_level |= log_level; in bpf_object__load_progs()
7858 bpf_program_record_relos(prog); in bpf_object__load_progs()
7860 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, in bpf_object__load_progs()
7861 obj->license, obj->kern_version, &prog->fd); in bpf_object__load_progs()
7863 pr_warn("prog '%s': failed to load: %d\n", prog->name, err); in bpf_object__load_progs()
7876 struct bpf_program *prog; in bpf_object_init_progs() local
7879 bpf_object__for_each_program(prog, obj) { in bpf_object_init_progs()
7880 prog->sec_def = find_sec_def(prog->sec_name); in bpf_object_init_progs()
7881 if (!prog->sec_def) { in bpf_object_init_progs()
7884 prog->name, prog->sec_name); in bpf_object_init_progs()
7888 prog->type = prog->sec_def->prog_type; in bpf_object_init_progs()
7889 prog->expected_attach_type = prog->sec_def->expected_attach_type; in bpf_object_init_progs()
7894 if (prog->sec_def->prog_setup_fn) { in bpf_object_init_progs()
7895 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); in bpf_object_init_progs()
7898 prog->name, err); in bpf_object_init_progs()
8456 struct bpf_program *prog = st_ops->progs[i]; in bpf_map_prepare_vdata() local
8460 if (!prog) in bpf_map_prepare_vdata()
8463 prog_fd = bpf_program__fd(prog); in bpf_map_prepare_vdata()
8615 int bpf_program__pin(struct bpf_program *prog, const char *path) in bpf_program__pin() argument
8620 if (prog->fd < 0) { in bpf_program__pin()
8621 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); in bpf_program__pin()
8633 if (bpf_obj_pin(prog->fd, path)) { in bpf_program__pin()
8636 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp); in bpf_program__pin()
8640 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); in bpf_program__pin()
8644 int bpf_program__unpin(struct bpf_program *prog, const char *path) in bpf_program__unpin() argument
8648 if (prog->fd < 0) { in bpf_program__unpin()
8649 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); in bpf_program__unpin()
8661 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); in bpf_program__unpin()
8885 struct bpf_program *prog; in bpf_object__pin_programs() local
8897 bpf_object__for_each_program(prog, obj) { in bpf_object__pin_programs()
8898 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__pin_programs()
8902 err = bpf_program__pin(prog, buf); in bpf_object__pin_programs()
8910 while ((prog = bpf_object__prev_program(obj, prog))) { in bpf_object__pin_programs()
8911 if (pathname_concat(buf, sizeof(buf), path, prog->name)) in bpf_object__pin_programs()
8914 bpf_program__unpin(prog, buf); in bpf_object__pin_programs()
8922 struct bpf_program *prog; in bpf_object__unpin_programs() local
8928 bpf_object__for_each_program(prog, obj) { in bpf_object__unpin_programs()
8931 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__unpin_programs()
8935 err = bpf_program__unpin(prog, buf); in bpf_object__unpin_programs()
9132 struct bpf_program *prog = prev; in bpf_object__next_program() local
9135 prog = __bpf_program__iter(prog, obj, true); in bpf_object__next_program()
9136 } while (prog && prog_is_subprog(obj, prog)); in bpf_object__next_program()
9138 return prog; in bpf_object__next_program()
9144 struct bpf_program *prog = next; in bpf_object__prev_program() local
9147 prog = __bpf_program__iter(prog, obj, false); in bpf_object__prev_program()
9148 } while (prog && prog_is_subprog(obj, prog)); in bpf_object__prev_program()
9150 return prog; in bpf_object__prev_program()
9153 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) in bpf_program__set_ifindex() argument
9155 prog->prog_ifindex = ifindex; in bpf_program__set_ifindex()
9158 const char *bpf_program__name(const struct bpf_program *prog) in bpf_program__name() argument
9160 return prog->name; in bpf_program__name()
9163 const char *bpf_program__section_name(const struct bpf_program *prog) in bpf_program__section_name() argument
9165 return prog->sec_name; in bpf_program__section_name()
9168 bool bpf_program__autoload(const struct bpf_program *prog) in bpf_program__autoload() argument
9170 return prog->autoload; in bpf_program__autoload()
9173 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) in bpf_program__set_autoload() argument
9175 if (prog->obj->loaded) in bpf_program__set_autoload()
9178 prog->autoload = autoload; in bpf_program__set_autoload()
9182 bool bpf_program__autoattach(const struct bpf_program *prog) in bpf_program__autoattach() argument
9184 return prog->autoattach; in bpf_program__autoattach()
9187 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) in bpf_program__set_autoattach() argument
9189 prog->autoattach = autoattach; in bpf_program__set_autoattach()
9192 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) in bpf_program__insns() argument
9194 return prog->insns; in bpf_program__insns()
9197 size_t bpf_program__insn_cnt(const struct bpf_program *prog) in bpf_program__insn_cnt() argument
9199 return prog->insns_cnt; in bpf_program__insn_cnt()
9202 int bpf_program__set_insns(struct bpf_program *prog, in bpf_program__set_insns() argument
9207 if (prog->obj->loaded) in bpf_program__set_insns()
9210 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); in bpf_program__set_insns()
9213 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); in bpf_program__set_insns()
9218 prog->insns = insns; in bpf_program__set_insns()
9219 prog->insns_cnt = new_insn_cnt; in bpf_program__set_insns()
9223 int bpf_program__fd(const struct bpf_program *prog) in bpf_program__fd() argument
9225 if (!prog) in bpf_program__fd()
9228 if (prog->fd < 0) in bpf_program__fd()
9231 return prog->fd; in bpf_program__fd()
9235 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
9237 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) in bpf_program__type() argument
9239 return prog->type; in bpf_program__type()
9248 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) in bpf_program__set_type() argument
9250 if (prog->obj->loaded) in bpf_program__set_type()
9254 if (prog->type == type) in bpf_program__set_type()
9257 prog->type = type; in bpf_program__set_type()
9265 if (prog->sec_def != &custom_fallback_def) in bpf_program__set_type()
9266 prog->sec_def = NULL; in bpf_program__set_type()
9271 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
9273 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) in bpf_program__expected_attach_type() argument
9275 return prog->expected_attach_type; in bpf_program__expected_attach_type()
9278 int bpf_program__set_expected_attach_type(struct bpf_program *prog, in bpf_program__set_expected_attach_type() argument
9281 if (prog->obj->loaded) in bpf_program__set_expected_attach_type()
9284 prog->expected_attach_type = type; in bpf_program__set_expected_attach_type()
9288 __u32 bpf_program__flags(const struct bpf_program *prog) in bpf_program__flags() argument
9290 return prog->prog_flags; in bpf_program__flags()
9293 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) in bpf_program__set_flags() argument
9295 if (prog->obj->loaded) in bpf_program__set_flags()
9298 prog->prog_flags = flags; in bpf_program__set_flags()
9302 __u32 bpf_program__log_level(const struct bpf_program *prog) in bpf_program__log_level() argument
9304 return prog->log_level; in bpf_program__log_level()
9307 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) in bpf_program__set_log_level() argument
9309 if (prog->obj->loaded) in bpf_program__set_log_level()
9312 prog->log_level = log_level; in bpf_program__set_log_level()
9316 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) in bpf_program__log_buf() argument
9318 *log_size = prog->log_size; in bpf_program__log_buf()
9319 return prog->log_buf; in bpf_program__log_buf()
9322 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) in bpf_program__set_log_buf() argument
9326 if (prog->log_size > UINT_MAX) in bpf_program__set_log_buf()
9328 if (prog->obj->loaded) in bpf_program__set_log_buf()
9331 prog->log_buf = log_buf; in bpf_program__set_log_buf()
9332 prog->log_size = log_size; in bpf_program__set_log_buf()
9345 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9346 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9347 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9348 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9349 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9350 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9351 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9352 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9353 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **lin…
9354 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9355 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9356 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9726 struct bpf_program *prog; in bpf_object__collect_st_ops_relos() local
9799 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); in bpf_object__collect_st_ops_relos()
9800 if (!prog) { in bpf_object__collect_st_ops_relos()
9807 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { in bpf_object__collect_st_ops_relos()
9809 map->name, prog->name); in bpf_object__collect_st_ops_relos()
9813 st_ops->progs[member_idx] = prog; in bpf_object__collect_st_ops_relos()
9821 *((struct bpf_program **)(st_ops->data + moff)) = prog; in bpf_object__collect_st_ops_relos()
9993 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, in libbpf_find_attach_btf_id() argument
9996 enum bpf_attach_type attach_type = prog->expected_attach_type; in libbpf_find_attach_btf_id()
9997 __u32 attach_prog_fd = prog->attach_prog_fd; in libbpf_find_attach_btf_id()
10001 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { in libbpf_find_attach_btf_id()
10003 pr_warn("prog '%s': attach program FD is not set\n", prog->name); in libbpf_find_attach_btf_id()
10009 prog->name, attach_prog_fd, attach_name, err); in libbpf_find_attach_btf_id()
10018 if (prog->obj->gen_loader) { in libbpf_find_attach_btf_id()
10019 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); in libbpf_find_attach_btf_id()
10023 err = find_kernel_btf_id(prog->obj, attach_name, in libbpf_find_attach_btf_id()
10029 prog->name, attach_name, err); in libbpf_find_attach_btf_id()
10565 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) in bpf_link__update_program() argument
10568 int prog_fd = bpf_program__fd(prog); in bpf_link__update_program()
10572 prog->name); in bpf_link__update_program()
10752 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, in bpf_program__attach_perf_event_opts() argument
10765 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10768 prog_fd = bpf_program__fd(prog); in bpf_program__attach_perf_event_opts()
10771 prog->name); in bpf_program__attach_perf_event_opts()
10783 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { in bpf_program__attach_perf_event_opts()
10791 prog->name, pfd, in bpf_program__attach_perf_event_opts()
10798 pr_warn("prog '%s': user context value is not supported\n", prog->name); in bpf_program__attach_perf_event_opts()
10806 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_perf_event_opts()
10809 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10817 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_perf_event_opts()
10829 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) in bpf_program__attach_perf_event() argument
10831 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); in bpf_program__attach_perf_event()
11153 bpf_program__attach_kprobe_opts(const struct bpf_program *prog, in bpf_program__attach_kprobe_opts() argument
11186 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_kprobe_opts()
11215 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
11220 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); in bpf_program__attach_kprobe_opts()
11225 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
11248 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, in bpf_program__attach_kprobe() argument
11256 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); in bpf_program__attach_kprobe()
11259 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, in bpf_program__attach_ksyscall() argument
11269 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { in bpf_program__attach_ksyscall()
11284 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); in bpf_program__attach_ksyscall()
11474 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, in bpf_program__attach_kprobe_multi_opts() argument
11495 prog_fd = bpf_program__fd(prog); in bpf_program__attach_kprobe_multi_opts()
11498 prog->name); in bpf_program__attach_kprobe_multi_opts()
11552 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_kprobe_multi_opts()
11565 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_kprobe() argument
11576 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) in attach_kprobe()
11579 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); in attach_kprobe()
11581 func_name = prog->sec_name + sizeof("kretprobe/") - 1; in attach_kprobe()
11583 func_name = prog->sec_name + sizeof("kprobe/") - 1; in attach_kprobe()
11597 *link = bpf_program__attach_kprobe_opts(prog, func, &opts); in attach_kprobe()
11602 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_ksyscall() argument
11610 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) in attach_ksyscall()
11613 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); in attach_ksyscall()
11615 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; in attach_ksyscall()
11617 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; in attach_ksyscall()
11619 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); in attach_ksyscall()
11623 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_kprobe_multi() argument
11633 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || in attach_kprobe_multi()
11634 strcmp(prog->sec_name, "kretprobe.multi") == 0) in attach_kprobe_multi()
11637 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); in attach_kprobe_multi()
11639 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; in attach_kprobe_multi()
11641 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; in attach_kprobe_multi()
11649 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); in attach_kprobe_multi()
11654 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, in attach_kprobe_session() argument
11665 if (strcmp(prog->sec_name, "kprobe.session") == 0) in attach_kprobe_session()
11668 spec = prog->sec_name + sizeof("kprobe.session/") - 1; in attach_kprobe_session()
11675 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); in attach_kprobe_session()
11680 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_uprobe_multi() argument
11688 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe_multi()
11697 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); in attach_uprobe_multi()
11701 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe_multi()
11702 prog->sec_name); in attach_uprobe_multi()
11936 bpf_program__attach_uprobe_multi(const struct bpf_program *prog, in bpf_program__attach_uprobe_multi() argument
11956 prog_fd = bpf_program__fd(prog); in bpf_program__attach_uprobe_multi()
11959 prog->name); in bpf_program__attach_uprobe_multi()
12001 prog->name, path, err); in bpf_program__attach_uprobe_multi()
12042 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); in bpf_program__attach_uprobe_multi()
12056 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, in bpf_program__attach_uprobe_opts() argument
12094 prog->name, binary_path, err); in bpf_program__attach_uprobe_opts()
12127 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_uprobe_opts()
12158 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
12164 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); in bpf_program__attach_uprobe_opts()
12169 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
12201 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_uprobe() argument
12210 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe()
12219 prog->name, prog->sec_name); in attach_uprobe()
12237 prog->name); in attach_uprobe()
12241 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); in attach_uprobe()
12245 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe()
12246 prog->sec_name); in attach_uprobe()
12256 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, in bpf_program__attach_uprobe() argument
12263 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); in bpf_program__attach_uprobe()
12266 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, in bpf_program__attach_usdt() argument
12272 struct bpf_object *obj = prog->obj; in bpf_program__attach_usdt()
12280 if (bpf_program__fd(prog) < 0) { in bpf_program__attach_usdt()
12282 prog->name); in bpf_program__attach_usdt()
12293 prog->name, binary_path, err); in bpf_program__attach_usdt()
12311 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, in bpf_program__attach_usdt()
12319 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_usdt() argument
12325 sec_name = bpf_program__section_name(prog); in attach_usdt()
12338 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, in attach_usdt()
12399 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, in bpf_program__attach_tracepoint_opts() argument
12417 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
12421 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); in bpf_program__attach_tracepoint_opts()
12426 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
12433 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, in bpf_program__attach_tracepoint() argument
12437 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); in bpf_program__attach_tracepoint()
12440 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_tp() argument
12447 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) in attach_tp()
12450 sec_name = strdup(prog->sec_name); in attach_tp()
12455 if (str_has_pfx(prog->sec_name, "tp/")) in attach_tp()
12467 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); in attach_tp()
12473 bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, in bpf_program__attach_raw_tracepoint_opts() argument
12485 prog_fd = bpf_program__fd(prog); in bpf_program__attach_raw_tracepoint_opts()
12487 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_raw_tracepoint_opts()
12503 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); in bpf_program__attach_raw_tracepoint_opts()
12510 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, in bpf_program__attach_raw_tracepoint() argument
12513 return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL); in bpf_program__attach_raw_tracepoint()
12516 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_raw_tp() argument
12532 if (!str_has_pfx(prog->sec_name, prefixes[i])) in attach_raw_tp()
12537 if (prog->sec_name[pfx_len] == '\0') in attach_raw_tp()
12540 if (prog->sec_name[pfx_len] != '/') in attach_raw_tp()
12543 tp_name = prog->sec_name + pfx_len + 1; in attach_raw_tp()
12549 prog->name, prog->sec_name); in attach_raw_tp()
12553 *link = bpf_program__attach_raw_tracepoint(prog, tp_name); in attach_raw_tp()
12558 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, in bpf_program__attach_btf_id() argument
12569 prog_fd = bpf_program__fd(prog); in bpf_program__attach_btf_id()
12571 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_btf_id()
12582 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); in bpf_program__attach_btf_id()
12587 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); in bpf_program__attach_btf_id()
12594 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) in bpf_program__attach_trace() argument
12596 return bpf_program__attach_btf_id(prog, NULL); in bpf_program__attach_trace()
12599 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, in bpf_program__attach_trace_opts() argument
12602 return bpf_program__attach_btf_id(prog, opts); in bpf_program__attach_trace_opts()
12605 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) in bpf_program__attach_lsm() argument
12607 return bpf_program__attach_btf_id(prog, NULL); in bpf_program__attach_lsm()
12610 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_trace() argument
12612 *link = bpf_program__attach_trace(prog); in attach_trace()
12616 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_lsm() argument
12618 *link = bpf_program__attach_lsm(prog); in attach_lsm()
12623 bpf_program_attach_fd(const struct bpf_program *prog, in bpf_program_attach_fd() argument
12632 prog_fd = bpf_program__fd(prog); in bpf_program_attach_fd()
12634 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program_attach_fd()
12643 attach_type = bpf_program__expected_attach_type(prog); in bpf_program_attach_fd()
12649 prog->name, target_name, in bpf_program_attach_fd()
12658 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) in bpf_program__attach_cgroup() argument
12660 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); in bpf_program__attach_cgroup()
12664 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) in bpf_program__attach_netns() argument
12666 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); in bpf_program__attach_netns()
12670 bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd) in bpf_program__attach_sockmap() argument
12672 return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL); in bpf_program__attach_sockmap()
12675 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) in bpf_program__attach_xdp() argument
12678 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); in bpf_program__attach_xdp()
12682 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, in bpf_program__attach_tcx() argument
12698 prog->name); in bpf_program__attach_tcx()
12703 prog->name); in bpf_program__attach_tcx()
12713 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); in bpf_program__attach_tcx()
12717 bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex, in bpf_program__attach_netkit() argument
12733 prog->name); in bpf_program__attach_netkit()
12738 prog->name); in bpf_program__attach_netkit()
12747 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts); in bpf_program__attach_netkit()
12750 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, in bpf_program__attach_freplace() argument
12758 prog->name); in bpf_program__attach_freplace()
12762 if (prog->type != BPF_PROG_TYPE_EXT) { in bpf_program__attach_freplace()
12764 prog->name); in bpf_program__attach_freplace()
12777 return bpf_program_attach_fd(prog, target_fd, "freplace", in bpf_program__attach_freplace()
12783 return bpf_program__attach_trace(prog); in bpf_program__attach_freplace()
12788 bpf_program__attach_iter(const struct bpf_program *prog, in bpf_program__attach_iter() argument
12803 prog_fd = bpf_program__fd(prog); in bpf_program__attach_iter()
12805 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_iter()
12820 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); in bpf_program__attach_iter()
12827 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_iter() argument
12829 *link = bpf_program__attach_iter(prog, NULL); in attach_iter()
12833 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, in bpf_program__attach_netfilter() argument
12843 prog_fd = bpf_program__fd(prog); in bpf_program__attach_netfilter()
12845 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_netfilter()
12867 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); in bpf_program__attach_netfilter()
12875 struct bpf_link *bpf_program__attach(const struct bpf_program *prog) in bpf_program__attach() argument
12880 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_program__attach()
12883 if (bpf_program__fd(prog) < 0) { in bpf_program__attach()
12885 prog->name); in bpf_program__attach()
12889 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); in bpf_program__attach()
13574 int bpf_program__set_attach_target(struct bpf_program *prog, in bpf_program__set_attach_target() argument
13580 if (!prog || attach_prog_fd < 0) in bpf_program__set_attach_target()
13583 if (prog->obj->loaded) in bpf_program__set_attach_target()
13590 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
13604 err = bpf_object__load_vmlinux_btf(prog->obj, true); in bpf_program__set_attach_target()
13607 err = find_kernel_btf_id(prog->obj, attach_func_name, in bpf_program__set_attach_target()
13608 prog->expected_attach_type, in bpf_program__set_attach_target()
13614 prog->attach_btf_id = btf_id; in bpf_program__set_attach_target()
13615 prog->attach_btf_obj_fd = btf_obj_fd; in bpf_program__set_attach_target()
13616 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
13756 struct bpf_program **prog = prog_skel->prog; in populate_skeleton_progs() local
13759 *prog = bpf_object__find_program_by_name(obj, name); in populate_skeleton_progs()
13760 if (!*prog) { in populate_skeleton_progs()
13931 struct bpf_program *prog = *prog_skel->prog; in bpf_object__attach_skeleton() local
13934 if (!prog->autoload || !prog->autoattach) in bpf_object__attach_skeleton()
13938 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_object__attach_skeleton()
13945 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); in bpf_object__attach_skeleton()
13948 bpf_program__name(prog), err); in bpf_object__attach_skeleton()