Lines Matching full:aux
100 struct bpf_prog_aux *aux; in bpf_prog_alloc_no_stats() local
108 aux = kzalloc(sizeof(*aux), bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); in bpf_prog_alloc_no_stats()
109 if (aux == NULL) { in bpf_prog_alloc_no_stats()
116 kfree(aux); in bpf_prog_alloc_no_stats()
121 fp->aux = aux; in bpf_prog_alloc_no_stats()
122 fp->aux->prog = fp; in bpf_prog_alloc_no_stats()
126 aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; in bpf_prog_alloc_no_stats()
129 INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); in bpf_prog_alloc_no_stats()
131 INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode); in bpf_prog_alloc_no_stats()
133 mutex_init(&fp->aux->used_maps_mutex); in bpf_prog_alloc_no_stats()
134 mutex_init(&fp->aux->dst_mutex); in bpf_prog_alloc_no_stats()
152 kfree(prog->aux); in bpf_prog_alloc()
169 if (!prog->aux->nr_linfo || !prog->jit_requested) in bpf_prog_alloc_jited_linfo()
172 prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo, in bpf_prog_alloc_jited_linfo()
173 sizeof(*prog->aux->jited_linfo), in bpf_prog_alloc_jited_linfo()
175 if (!prog->aux->jited_linfo) in bpf_prog_alloc_jited_linfo()
183 if (prog->aux->jited_linfo && in bpf_prog_jit_attempt_done()
184 (!prog->jited || !prog->aux->jited_linfo[0])) { in bpf_prog_jit_attempt_done()
185 kvfree(prog->aux->jited_linfo); in bpf_prog_jit_attempt_done()
186 prog->aux->jited_linfo = NULL; in bpf_prog_jit_attempt_done()
189 kfree(prog->aux->kfunc_tab); in bpf_prog_jit_attempt_done()
190 prog->aux->kfunc_tab = NULL; in bpf_prog_jit_attempt_done()
208 * The prog's idx to prog->aux->linfo and jited_linfo
224 if (!prog->aux->jited_linfo || prog->aux->func_idx > prog->aux->func_cnt) in bpf_prog_fill_jited_linfo()
228 linfo_idx = prog->aux->linfo_idx; in bpf_prog_fill_jited_linfo()
229 linfo = &prog->aux->linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
233 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
236 nr_linfo = prog->aux->nr_linfo - linfo_idx; in bpf_prog_fill_jited_linfo()
262 fp->aux->prog = fp; in bpf_prog_realloc()
264 /* We keep fp->aux from fp_old around in the new in bpf_prog_realloc()
267 fp_old->aux = NULL; in bpf_prog_realloc()
278 if (fp->aux) { in __bpf_prog_free()
279 mutex_destroy(&fp->aux->used_maps_mutex); in __bpf_prog_free()
280 mutex_destroy(&fp->aux->dst_mutex); in __bpf_prog_free()
281 kfree(fp->aux->poke_tab); in __bpf_prog_free()
282 kfree(fp->aux); in __bpf_prog_free()
463 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo()
467 linfo = prog->aux->linfo; in bpf_adj_linfo()
555 for (i = 0; i < fp->aux->real_func_cnt; i++) in bpf_prog_kallsyms_del_subprogs()
556 bpf_prog_kallsyms_del(fp->aux->func[i]); in bpf_prog_kallsyms_del_subprogs()
578 prog->aux->ksym.start = (unsigned long) prog->bpf_func; in bpf_prog_ksym_set_addr()
579 prog->aux->ksym.end = prog->aux->ksym.start + prog->jited_len; in bpf_prog_ksym_set_addr()
585 char *sym = prog->aux->ksym.name; in bpf_prog_ksym_set_name()
599 sizeof(prog->aux->name) > KSYM_NAME_LEN); in bpf_prog_ksym_set_name()
604 /* prog->aux->name will be ignored if full btf name is available */ in bpf_prog_ksym_set_name()
605 if (prog->aux->func_info_cnt && prog->aux->func_idx < prog->aux->func_info_cnt) { in bpf_prog_ksym_set_name()
606 type = btf_type_by_id(prog->aux->btf, in bpf_prog_ksym_set_name()
607 prog->aux->func_info[prog->aux->func_idx].type_id); in bpf_prog_ksym_set_name()
608 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); in bpf_prog_ksym_set_name()
613 if (prog->aux->name[0]) in bpf_prog_ksym_set_name()
614 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); in bpf_prog_ksym_set_name()
691 !bpf_token_capable(fp->aux->token, CAP_BPF)) in bpf_prog_kallsyms_add()
696 fp->aux->ksym.prog = true; in bpf_prog_kallsyms_add()
698 bpf_ksym_add(&fp->aux->ksym); in bpf_prog_kallsyms_add()
708 snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN, in bpf_prog_kallsyms_add()
709 "__cfi_%s", fp->aux->ksym.name); in bpf_prog_kallsyms_add()
711 fp->aux->ksym_prefix.start = (unsigned long) fp->bpf_func - 16; in bpf_prog_kallsyms_add()
712 fp->aux->ksym_prefix.end = (unsigned long) fp->bpf_func; in bpf_prog_kallsyms_add()
714 bpf_ksym_add(&fp->aux->ksym_prefix); in bpf_prog_kallsyms_add()
723 bpf_ksym_del(&fp->aux->ksym); in bpf_prog_kallsyms_del()
727 bpf_ksym_del(&fp->aux->ksym_prefix); in bpf_prog_kallsyms_del()
792 if (!prog->aux->num_exentries) in search_bpf_extables()
795 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr); in search_bpf_extables()
832 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; in bpf_jit_add_poke_descriptor()
834 u32 slot = prog->aux->size_poke_tab; in bpf_jit_add_poke_descriptor()
857 prog->aux->size_poke_tab = size; in bpf_jit_add_poke_descriptor()
858 prog->aux->poke_tab = tab; in bpf_jit_add_poke_descriptor()
1261 * can pick them up from prog->aux. in bpf_jit_get_func_addr()
1265 else if (prog->aux->func && in bpf_jit_get_func_addr()
1266 off >= 0 && off < prog->aux->real_func_cnt) in bpf_jit_get_func_addr()
1267 addr = (u8 *)prog->aux->func[off]->bpf_func; in bpf_jit_get_func_addr()
1289 const struct bpf_insn *aux, in bpf_jit_blind_insn() argument
1398 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); in bpf_jit_blind_insn()
1401 *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); in bpf_jit_blind_insn()
1404 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); in bpf_jit_blind_insn()
1408 *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); in bpf_jit_blind_insn()
1432 /* aux->prog still points to the fp_other one, so in bpf_prog_clone_create()
1444 /* aux was stolen by the other clone, so we cannot free in bpf_prog_clone_free()
1451 fp->aux = NULL; in bpf_prog_clone_free()
1459 /* We have to repoint aux->prog to self, as we don't in bpf_jit_prog_release_other()
1462 fp->aux->prog = fp; in bpf_jit_prog_release_other()
1468 struct bpf_insn insn_buff[16], aux[2]; in bpf_jit_blind_constants() local
1501 memcpy(aux, insn, sizeof(aux)); in bpf_jit_blind_constants()
1503 rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, in bpf_jit_blind_constants()
1504 clone->aux->verifier_zext); in bpf_jit_blind_constants()
1510 /* Patching may have repointed aux->prog during in bpf_jit_blind_constants()
2305 struct bpf_prog_aux *aux = fp->aux; in bpf_prog_map_compatible() local
2315 if (bpf_prog_is_dev_bound(aux)) in bpf_prog_map_compatible()
2325 map->owner.xdp_has_frags = aux->xdp_has_frags; in bpf_prog_map_compatible()
2326 map->owner.attach_func_proto = aux->attach_func_proto; in bpf_prog_map_compatible()
2331 map->owner.xdp_has_frags == aux->xdp_has_frags; in bpf_prog_map_compatible()
2333 map->owner.attach_func_proto != aux->attach_func_proto) { in bpf_prog_map_compatible()
2353 struct bpf_prog_aux *aux = fp->aux; in bpf_check_tail_call() local
2356 mutex_lock(&aux->used_maps_mutex); in bpf_check_tail_call()
2357 for (i = 0; i < aux->used_map_cnt; i++) { in bpf_check_tail_call()
2358 struct bpf_map *map = aux->used_maps[i]; in bpf_check_tail_call()
2370 mutex_unlock(&aux->used_maps_mutex); in bpf_check_tail_call()
2377 u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); in bpf_prog_select_func()
2418 if (!bpf_prog_is_offloaded(fp->aux)) { in bpf_prog_select_runtime()
2541 prog_ids[i] = item->prog->aux->id; in bpf_prog_array_copy_core()
2735 void __bpf_free_used_maps(struct bpf_prog_aux *aux, in __bpf_free_used_maps() argument
2742 sleepable = aux->prog->sleepable; in __bpf_free_used_maps()
2746 map->ops->map_poke_untrack(map, aux); in __bpf_free_used_maps()
2753 static void bpf_free_used_maps(struct bpf_prog_aux *aux) in bpf_free_used_maps() argument
2755 __bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt); in bpf_free_used_maps()
2756 kfree(aux->used_maps); in bpf_free_used_maps()
2774 static void bpf_free_used_btfs(struct bpf_prog_aux *aux) in bpf_free_used_btfs() argument
2776 __bpf_free_used_btfs(aux->used_btfs, aux->used_btf_cnt); in bpf_free_used_btfs()
2777 kfree(aux->used_btfs); in bpf_free_used_btfs()
2782 struct bpf_prog_aux *aux; in bpf_prog_free_deferred() local
2785 aux = container_of(work, struct bpf_prog_aux, work); in bpf_prog_free_deferred()
2787 bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); in bpf_prog_free_deferred()
2790 if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) in bpf_prog_free_deferred()
2791 bpf_cgroup_atype_put(aux->cgroup_atype); in bpf_prog_free_deferred()
2793 bpf_free_used_maps(aux); in bpf_prog_free_deferred()
2794 bpf_free_used_btfs(aux); in bpf_prog_free_deferred()
2795 if (bpf_prog_is_dev_bound(aux)) in bpf_prog_free_deferred()
2796 bpf_prog_dev_bound_destroy(aux->prog); in bpf_prog_free_deferred()
2798 if (aux->prog->has_callchain_buf) in bpf_prog_free_deferred()
2801 if (aux->dst_trampoline) in bpf_prog_free_deferred()
2802 bpf_trampoline_put(aux->dst_trampoline); in bpf_prog_free_deferred()
2803 for (i = 0; i < aux->real_func_cnt; i++) { in bpf_prog_free_deferred()
2808 aux->func[i]->aux->poke_tab = NULL; in bpf_prog_free_deferred()
2809 bpf_jit_free(aux->func[i]); in bpf_prog_free_deferred()
2811 if (aux->real_func_cnt) { in bpf_prog_free_deferred()
2812 kfree(aux->func); in bpf_prog_free_deferred()
2813 bpf_prog_unlock_free(aux->prog); in bpf_prog_free_deferred()
2815 bpf_jit_free(aux->prog); in bpf_prog_free_deferred()
2821 struct bpf_prog_aux *aux = fp->aux; in bpf_prog_free() local
2823 if (aux->dst_prog) in bpf_prog_free()
2824 bpf_prog_put(aux->dst_prog); in bpf_prog_free()
2825 bpf_token_put(aux->token); in bpf_prog_free()
2826 INIT_WORK(&aux->work, bpf_prog_free_deferred); in bpf_prog_free()
2827 schedule_work(&aux->work); in bpf_prog_free()