Lines Matching full:gen
45 static int blob_fd_array_off(struct bpf_gen *gen, int index) in blob_fd_array_off() argument
47 return gen->fd_array + index * sizeof(int); in blob_fd_array_off()
50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) in realloc_insn_buf() argument
52 size_t off = gen->insn_cur - gen->insn_start; in realloc_insn_buf()
55 if (gen->error) in realloc_insn_buf()
56 return gen->error; in realloc_insn_buf()
58 gen->error = -ERANGE; in realloc_insn_buf()
61 insn_start = realloc(gen->insn_start, off + size); in realloc_insn_buf()
63 gen->error = -ENOMEM; in realloc_insn_buf()
64 free(gen->insn_start); in realloc_insn_buf()
65 gen->insn_start = NULL; in realloc_insn_buf()
68 gen->insn_start = insn_start; in realloc_insn_buf()
69 gen->insn_cur = insn_start + off; in realloc_insn_buf()
73 static int realloc_data_buf(struct bpf_gen *gen, __u32 size) in realloc_data_buf() argument
75 size_t off = gen->data_cur - gen->data_start; in realloc_data_buf()
78 if (gen->error) in realloc_data_buf()
79 return gen->error; in realloc_data_buf()
81 gen->error = -ERANGE; in realloc_data_buf()
84 data_start = realloc(gen->data_start, off + size); in realloc_data_buf()
86 gen->error = -ENOMEM; in realloc_data_buf()
87 free(gen->data_start); in realloc_data_buf()
88 gen->data_start = NULL; in realloc_data_buf()
91 gen->data_start = data_start; in realloc_data_buf()
92 gen->data_cur = data_start + off; in realloc_data_buf()
96 static void emit(struct bpf_gen *gen, struct bpf_insn insn) in emit() argument
98 if (realloc_insn_buf(gen, sizeof(insn))) in emit()
100 memcpy(gen->insn_cur, &insn, sizeof(insn)); in emit()
101 gen->insn_cur += sizeof(insn); in emit()
104 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) in emit2() argument
106 emit(gen, insn1); in emit2()
107 emit(gen, insn2); in emit2()
110 static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
111 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
113 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) in bpf_gen__init() argument
118 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); in bpf_gen__init()
119 gen->log_level = log_level; in bpf_gen__init()
121 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); in bpf_gen__init()
124 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); in bpf_gen__init()
125 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); in bpf_gen__init()
126 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); in bpf_gen__init()
127 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); in bpf_gen__init()
128 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); in bpf_gen__init()
133 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, in bpf_gen__init()
139 nr_maps * (6 + (gen->log_level ? 6 : 0)))); in bpf_gen__init()
142 gen->cleanup_label = gen->insn_cur - gen->insn_start; in bpf_gen__init()
145 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); in bpf_gen__init()
146 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); in bpf_gen__init()
147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in bpf_gen__init()
150 emit_sys_close_blob(gen, blob_fd_array_off(gen, i)); in bpf_gen__init()
152 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); in bpf_gen__init()
153 emit(gen, BPF_EXIT_INSN()); in bpf_gen__init()
156 static int add_data(struct bpf_gen *gen, const void *data, __u32 size) in add_data() argument
162 if (realloc_data_buf(gen, size8)) in add_data()
164 prev = gen->data_cur; in add_data()
166 memcpy(gen->data_cur, data, size); in add_data()
167 memcpy(gen->data_cur + size, &zero, size8 - size); in add_data()
169 memset(gen->data_cur, 0, size8); in add_data()
171 gen->data_cur += size8; in add_data()
172 return prev - gen->data_start; in add_data()
178 static int add_map_fd(struct bpf_gen *gen) in add_map_fd() argument
180 if (gen->nr_maps == MAX_USED_MAPS) { in add_map_fd()
182 gen->error = -E2BIG; in add_map_fd()
185 return gen->nr_maps++; in add_map_fd()
188 static int add_kfunc_btf_fd(struct bpf_gen *gen) in add_kfunc_btf_fd() argument
192 if (gen->nr_fd_array == MAX_KFUNC_DESCS) { in add_kfunc_btf_fd()
193 cur = add_data(gen, NULL, sizeof(int)); in add_kfunc_btf_fd()
194 return (cur - gen->fd_array) / sizeof(int); in add_kfunc_btf_fd()
196 return MAX_USED_MAPS + gen->nr_fd_array++; in add_kfunc_btf_fd()
211 static void emit_rel_store(struct bpf_gen *gen, int off, int data) in emit_rel_store() argument
213 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
215 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
217 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); in emit_rel_store()
220 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off) in move_blob2blob() argument
222 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2blob()
224 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0)); in move_blob2blob()
225 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2blob()
227 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_blob2blob()
230 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off) in move_blob2ctx() argument
232 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2ctx()
234 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0)); in move_blob2ctx()
235 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); in move_blob2ctx()
238 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, in move_ctx2blob() argument
241 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); in move_ctx2blob()
246 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); in move_ctx2blob()
247 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_ctx2blob()
249 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_ctx2blob()
252 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) in move_stack2blob() argument
254 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2blob()
255 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_stack2blob()
257 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_stack2blob()
260 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) in move_stack2ctx() argument
262 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2ctx()
263 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); in move_stack2ctx()
266 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) in emit_sys_bpf() argument
268 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); in emit_sys_bpf()
269 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_bpf()
271 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); in emit_sys_bpf()
272 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); in emit_sys_bpf()
274 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_sys_bpf()
282 static void emit_check_err(struct bpf_gen *gen) in emit_check_err() argument
284 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; in emit_check_err()
290 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); in emit_check_err()
292 gen->error = -ERANGE; in emit_check_err()
293 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); in emit_check_err()
298 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, in emit_debug() argument
304 if (!gen->log_level) in emit_debug()
314 addr = add_data(gen, buf, len); in emit_debug()
316 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_debug()
318 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_debug()
320 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); in emit_debug()
322 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); in emit_debug()
323 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); in emit_debug()
326 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) in debug_regs() argument
331 emit_debug(gen, reg1, reg2, fmt, args); in debug_regs()
335 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) in debug_ret() argument
340 emit_debug(gen, BPF_REG_7, -1, fmt, args); in debug_ret()
344 static void __emit_sys_close(struct bpf_gen *gen) in __emit_sys_close() argument
346 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, in __emit_sys_close()
350 2 + (gen->log_level ? 6 : 0))); in __emit_sys_close()
351 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); in __emit_sys_close()
352 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in __emit_sys_close()
353 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); in __emit_sys_close()
356 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) in emit_sys_close_stack() argument
358 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); in emit_sys_close_stack()
359 __emit_sys_close(gen); in emit_sys_close_stack()
362 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) in emit_sys_close_blob() argument
364 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_close_blob()
366 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); in emit_sys_close_blob()
367 __emit_sys_close(gen); in emit_sys_close_blob()
370 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) in bpf_gen__finish() argument
374 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { in bpf_gen__finish()
376 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); in bpf_gen__finish()
377 gen->error = -EFAULT; in bpf_gen__finish()
378 return gen->error; in bpf_gen__finish()
380 emit_sys_close_stack(gen, stack_off(btf_fd)); in bpf_gen__finish()
381 for (i = 0; i < gen->nr_progs; i++) in bpf_gen__finish()
382 move_stack2ctx(gen, in bpf_gen__finish()
384 sizeof(struct bpf_map_desc) * gen->nr_maps + in bpf_gen__finish()
388 for (i = 0; i < gen->nr_maps; i++) in bpf_gen__finish()
389 move_blob2ctx(gen, in bpf_gen__finish()
393 blob_fd_array_off(gen, i)); in bpf_gen__finish()
394 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); in bpf_gen__finish()
395 emit(gen, BPF_EXIT_INSN()); in bpf_gen__finish()
396 pr_debug("gen: finish %d\n", gen->error); in bpf_gen__finish()
397 if (!gen->error) { in bpf_gen__finish()
398 struct gen_loader_opts *opts = gen->opts; in bpf_gen__finish()
400 opts->insns = gen->insn_start; in bpf_gen__finish()
401 opts->insns_sz = gen->insn_cur - gen->insn_start; in bpf_gen__finish()
402 opts->data = gen->data_start; in bpf_gen__finish()
403 opts->data_sz = gen->data_cur - gen->data_start; in bpf_gen__finish()
405 return gen->error; in bpf_gen__finish()
408 void bpf_gen__free(struct bpf_gen *gen) in bpf_gen__free() argument
410 if (!gen) in bpf_gen__free()
412 free(gen->data_start); in bpf_gen__free()
413 free(gen->insn_start); in bpf_gen__free()
414 free(gen); in bpf_gen__free()
417 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, in bpf_gen__load_btf() argument
425 pr_debug("gen: load_btf: size %d\n", btf_raw_size); in bpf_gen__load_btf()
426 btf_data = add_data(gen, btf_raw_data, btf_raw_size); in bpf_gen__load_btf()
429 btf_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__load_btf()
432 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, in bpf_gen__load_btf()
434 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, in bpf_gen__load_btf()
436 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, in bpf_gen__load_btf()
439 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); in bpf_gen__load_btf()
441 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); in bpf_gen__load_btf()
442 debug_ret(gen, "btf_load size %d", btf_raw_size); in bpf_gen__load_btf()
443 emit_check_err(gen); in bpf_gen__load_btf()
445 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); in bpf_gen__load_btf()
448 void bpf_gen__map_create(struct bpf_gen *gen, in bpf_gen__map_create() argument
473 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n", in bpf_gen__map_create()
476 map_create_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_create()
479 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, in bpf_gen__map_create()
484 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, in bpf_gen__map_create()
493 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, in bpf_gen__map_create()
499 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); in bpf_gen__map_create()
500 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", in bpf_gen__map_create()
503 emit_check_err(gen); in bpf_gen__map_create()
510 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__map_create()
512 } else if (map_idx != gen->nr_maps) { in bpf_gen__map_create()
513 gen->error = -EDOM; /* internal bug */ in bpf_gen__map_create()
516 /* add_map_fd does gen->nr_maps++ */ in bpf_gen__map_create()
517 idx = add_map_fd(gen); in bpf_gen__map_create()
518 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__map_create()
519 0, 0, 0, blob_fd_array_off(gen, idx))); in bpf_gen__map_create()
520 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); in bpf_gen__map_create()
523 emit_sys_close_stack(gen, stack_off(inner_map_fd)); in bpf_gen__map_create()
526 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, in bpf_gen__record_attach_target() argument
533 gen->attach_kind = kind; in bpf_gen__record_attach_target()
534 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", in bpf_gen__record_attach_target()
536 if (ret >= sizeof(gen->attach_target)) in bpf_gen__record_attach_target()
537 gen->error = -ENOSPC; in bpf_gen__record_attach_target()
540 static void emit_find_attach_target(struct bpf_gen *gen) in emit_find_attach_target() argument
542 int name, len = strlen(gen->attach_target) + 1; in emit_find_attach_target()
544 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); in emit_find_attach_target()
545 name = add_data(gen, gen->attach_target, len); in emit_find_attach_target()
547 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_find_attach_target()
549 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_find_attach_target()
550 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); in emit_find_attach_target()
551 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_find_attach_target()
552 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_find_attach_target()
553 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_find_attach_target()
554 debug_ret(gen, "find_by_name_kind(%s,%d)", in emit_find_attach_target()
555 gen->attach_target, gen->attach_kind); in emit_find_attach_target()
556 emit_check_err(gen); in emit_find_attach_target()
562 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, in bpf_gen__record_extern() argument
567 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); in bpf_gen__record_extern()
569 gen->error = -ENOMEM; in bpf_gen__record_extern()
572 gen->relos = relo; in bpf_gen__record_extern()
573 relo += gen->relo_cnt; in bpf_gen__record_extern()
580 gen->relo_cnt++; in bpf_gen__record_extern()
584 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) in get_ksym_desc() argument
589 for (i = 0; i < gen->nr_ksyms; i++) { in get_ksym_desc()
590 kdesc = &gen->ksyms[i]; in get_ksym_desc()
597 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc)); in get_ksym_desc()
599 gen->error = -ENOMEM; in get_ksym_desc()
602 gen->ksyms = kdesc; in get_ksym_desc()
603 kdesc = &gen->ksyms[gen->nr_ksyms++]; in get_ksym_desc()
616 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo) in emit_bpf_find_by_name_kind() argument
620 name_off = add_data(gen, relo->name, len); in emit_bpf_find_by_name_kind()
621 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_find_by_name_kind()
623 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_bpf_find_by_name_kind()
624 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); in emit_bpf_find_by_name_kind()
625 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_bpf_find_by_name_kind()
626 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_bpf_find_by_name_kind()
627 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_bpf_find_by_name_kind()
628 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); in emit_bpf_find_by_name_kind()
635 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo) in emit_bpf_kallsyms_lookup_name() argument
639 name_off = add_data(gen, relo->name, len); in emit_bpf_kallsyms_lookup_name()
640 res_off = add_data(gen, NULL, 8); /* res is u64 */ in emit_bpf_kallsyms_lookup_name()
641 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_kallsyms_lookup_name()
643 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_bpf_kallsyms_lookup_name()
644 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); in emit_bpf_kallsyms_lookup_name()
645 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_kallsyms_lookup_name()
647 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4)); in emit_bpf_kallsyms_lookup_name()
648 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name)); in emit_bpf_kallsyms_lookup_name()
649 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0)); in emit_bpf_kallsyms_lookup_name()
650 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_bpf_kallsyms_lookup_name()
651 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind); in emit_bpf_kallsyms_lookup_name()
664 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) in emit_relo_kfunc_btf() argument
669 kdesc = get_ksym_desc(gen, relo); in emit_relo_kfunc_btf()
674 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_kfunc_btf()
676 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2, in emit_relo_kfunc_btf()
682 emit_bpf_find_by_name_kind(gen, relo); in emit_relo_kfunc_btf()
684 emit_check_err(gen); in emit_relo_kfunc_btf()
686 btf_fd_idx = add_kfunc_btf_fd(gen); in emit_relo_kfunc_btf()
690 gen->error = -E2BIG; in emit_relo_kfunc_btf()
695 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); in emit_relo_kfunc_btf()
697 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); in emit_relo_kfunc_btf()
698 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); in emit_relo_kfunc_btf()
700 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10)); in emit_relo_kfunc_btf()
702 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); in emit_relo_kfunc_btf()
704 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); in emit_relo_kfunc_btf()
705 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); in emit_relo_kfunc_btf()
707 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo_kfunc_btf()
708 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); in emit_relo_kfunc_btf()
710 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); in emit_relo_kfunc_btf()
712 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2)); in emit_relo_kfunc_btf()
714 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); in emit_relo_kfunc_btf()
716 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); in emit_relo_kfunc_btf()
718 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); in emit_relo_kfunc_btf()
720 if (!gen->log_level) in emit_relo_kfunc_btf()
722 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, in emit_relo_kfunc_btf()
724 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, in emit_relo_kfunc_btf()
726 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d", in emit_relo_kfunc_btf()
728 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo_kfunc_btf()
729 0, 0, 0, blob_fd_array_off(gen, kdesc->off))); in emit_relo_kfunc_btf()
730 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0)); in emit_relo_kfunc_btf()
731 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd", in emit_relo_kfunc_btf()
735 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo, in emit_ksym_relo_log() argument
738 if (!gen->log_level) in emit_ksym_relo_log()
740 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, in emit_ksym_relo_log()
742 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) + in emit_ksym_relo_log()
744 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d", in emit_ksym_relo_log()
746 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); in emit_ksym_relo_log()
747 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg", in emit_ksym_relo_log()
754 static void emit_relo_ksym_typeless(struct bpf_gen *gen, in emit_relo_ksym_typeless() argument
759 kdesc = get_ksym_desc(gen, relo); in emit_relo_ksym_typeless()
764 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_typeless()
766 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_typeless()
774 emit_bpf_kallsyms_lookup_name(gen, relo); in emit_relo_ksym_typeless()
775 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1)); in emit_relo_ksym_typeless()
776 emit_check_err(gen); in emit_relo_ksym_typeless()
778 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm))); in emit_relo_ksym_typeless()
780 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); in emit_relo_ksym_typeless()
781 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, in emit_relo_ksym_typeless()
784 emit_ksym_relo_log(gen, relo, kdesc->ref); in emit_relo_ksym_typeless()
801 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) in emit_relo_ksym_btf() argument
806 kdesc = get_ksym_desc(gen, relo); in emit_relo_ksym_btf()
811 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_btf()
813 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_btf()
818 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3)); in emit_relo_ksym_btf()
823 emit_bpf_find_by_name_kind(gen, relo); in emit_relo_ksym_btf()
825 emit_check_err(gen); in emit_relo_ksym_btf()
827 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); in emit_relo_ksym_btf()
829 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); in emit_relo_ksym_btf()
830 …emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0… in emit_relo_ksym_btf()
832 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); in emit_relo_ksym_btf()
834 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); in emit_relo_ksym_btf()
836 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in emit_relo_ksym_btf()
837 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, in emit_relo_ksym_btf()
840 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3)); in emit_relo_ksym_btf()
844 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); in emit_relo_ksym_btf()
845 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask)); in emit_relo_ksym_btf()
846 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code))); in emit_relo_ksym_btf()
848 emit_ksym_relo_log(gen, relo, kdesc->ref); in emit_relo_ksym_btf()
851 void bpf_gen__record_relo_core(struct bpf_gen *gen, in bpf_gen__record_relo_core() argument
856 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos)); in bpf_gen__record_relo_core()
858 gen->error = -ENOMEM; in bpf_gen__record_relo_core()
861 gen->core_relos = relos; in bpf_gen__record_relo_core()
862 relos += gen->core_relo_cnt; in bpf_gen__record_relo_core()
864 gen->core_relo_cnt++; in bpf_gen__record_relo_core()
867 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) in emit_relo() argument
871 pr_debug("gen: emit_relo (%d): %s at %d %s\n", in emit_relo()
874 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn)); in emit_relo()
877 emit_relo_ksym_typeless(gen, relo, insn); in emit_relo()
879 emit_relo_ksym_btf(gen, relo, insn); in emit_relo()
881 emit_relo_kfunc_btf(gen, relo, insn); in emit_relo()
885 static void emit_relos(struct bpf_gen *gen, int insns) in emit_relos() argument
889 for (i = 0; i < gen->relo_cnt; i++) in emit_relos()
890 emit_relo(gen, gen->relos + i, insns); in emit_relos()
893 static void cleanup_core_relo(struct bpf_gen *gen) in cleanup_core_relo() argument
895 if (!gen->core_relo_cnt) in cleanup_core_relo()
897 free(gen->core_relos); in cleanup_core_relo()
898 gen->core_relo_cnt = 0; in cleanup_core_relo()
899 gen->core_relos = NULL; in cleanup_core_relo()
902 static void cleanup_relos(struct bpf_gen *gen, int insns) in cleanup_relos() argument
907 for (i = 0; i < gen->nr_ksyms; i++) { in cleanup_relos()
908 kdesc = &gen->ksyms[i]; in cleanup_relos()
914 emit_sys_close_blob(gen, insn); in cleanup_relos()
916 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off)); in cleanup_relos()
918 gen->nr_fd_array--; in cleanup_relos()
921 if (gen->nr_ksyms) { in cleanup_relos()
922 free(gen->ksyms); in cleanup_relos()
923 gen->nr_ksyms = 0; in cleanup_relos()
924 gen->ksyms = NULL; in cleanup_relos()
926 if (gen->relo_cnt) { in cleanup_relos()
927 free(gen->relos); in cleanup_relos()
928 gen->relo_cnt = 0; in cleanup_relos()
929 gen->relos = NULL; in cleanup_relos()
931 cleanup_core_relo(gen); in cleanup_relos()
934 void bpf_gen__prog_load(struct bpf_gen *gen, in bpf_gen__prog_load() argument
944 pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n", in bpf_gen__prog_load()
947 license_off = add_data(gen, license, strlen(license) + 1); in bpf_gen__prog_load()
949 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); in bpf_gen__prog_load()
961 func_info = add_data(gen, load_attr->func_info, in bpf_gen__prog_load()
966 line_info = add_data(gen, load_attr->line_info, in bpf_gen__prog_load()
970 attr.core_relo_cnt = gen->core_relo_cnt; in bpf_gen__prog_load()
971 core_relos = add_data(gen, gen->core_relos, in bpf_gen__prog_load()
975 prog_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__prog_load()
978 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off); in bpf_gen__prog_load()
981 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off); in bpf_gen__prog_load()
984 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); in bpf_gen__prog_load()
987 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); in bpf_gen__prog_load()
990 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos); in bpf_gen__prog_load()
993 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); in bpf_gen__prog_load()
996 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, in bpf_gen__prog_load()
998 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, in bpf_gen__prog_load()
1000 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, in bpf_gen__prog_load()
1003 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, in bpf_gen__prog_load()
1005 if (gen->attach_kind) { in bpf_gen__prog_load()
1006 emit_find_attach_target(gen); in bpf_gen__prog_load()
1008 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__prog_load()
1010 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
1012 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in bpf_gen__prog_load()
1013 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
1016 emit_relos(gen, insns_off); in bpf_gen__prog_load()
1018 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); in bpf_gen__prog_load()
1019 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); in bpf_gen__prog_load()
1021 cleanup_relos(gen, insns_off); in bpf_gen__prog_load()
1022 if (gen->attach_kind) { in bpf_gen__prog_load()
1023 emit_sys_close_blob(gen, in bpf_gen__prog_load()
1025 gen->attach_kind = 0; in bpf_gen__prog_load()
1027 emit_check_err(gen); in bpf_gen__prog_load()
1029 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__prog_load()
1030 stack_off(prog_fd[gen->nr_progs]))); in bpf_gen__prog_load()
1031 gen->nr_progs++; in bpf_gen__prog_load()
1034 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, in bpf_gen__map_update_elem() argument
1043 pr_debug("gen: map_update_elem: idx %d\n", map_idx); in bpf_gen__map_update_elem()
1045 value = add_data(gen, pvalue, value_size); in bpf_gen__map_update_elem()
1046 key = add_data(gen, &zero, sizeof(zero)); in bpf_gen__map_update_elem()
1055 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, in bpf_gen__map_update_elem()
1059 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); in bpf_gen__map_update_elem()
1060 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__map_update_elem()
1062 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); in bpf_gen__map_update_elem()
1063 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, in bpf_gen__map_update_elem()
1065 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); in bpf_gen__map_update_elem()
1066 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); in bpf_gen__map_update_elem()
1067 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); in bpf_gen__map_update_elem()
1068 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); in bpf_gen__map_update_elem()
1070 map_update_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_update_elem()
1071 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, in bpf_gen__map_update_elem()
1072 blob_fd_array_off(gen, map_idx)); in bpf_gen__map_update_elem()
1073 emit_rel_store(gen, attr_field(map_update_attr, key), key); in bpf_gen__map_update_elem()
1074 emit_rel_store(gen, attr_field(map_update_attr, value), value); in bpf_gen__map_update_elem()
1076 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); in bpf_gen__map_update_elem()
1077 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); in bpf_gen__map_update_elem()
1078 emit_check_err(gen); in bpf_gen__map_update_elem()
1081 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, in bpf_gen__populate_outer_map() argument
1089 pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n", in bpf_gen__populate_outer_map()
1092 key = add_data(gen, &slot, sizeof(slot)); in bpf_gen__populate_outer_map()
1094 map_update_attr = add_data(gen, &attr, attr_size); in bpf_gen__populate_outer_map()
1095 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, in bpf_gen__populate_outer_map()
1096 blob_fd_array_off(gen, outer_map_idx)); in bpf_gen__populate_outer_map()
1097 emit_rel_store(gen, attr_field(map_update_attr, key), key); in bpf_gen__populate_outer_map()
1098 emit_rel_store(gen, attr_field(map_update_attr, value), in bpf_gen__populate_outer_map()
1099 blob_fd_array_off(gen, inner_map_idx)); in bpf_gen__populate_outer_map()
1102 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); in bpf_gen__populate_outer_map()
1103 debug_ret(gen, "populate_outer_map outer %d key %d inner %d", in bpf_gen__populate_outer_map()
1105 emit_check_err(gen); in bpf_gen__populate_outer_map()
1108 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) in bpf_gen__map_freeze() argument
1115 pr_debug("gen: map_freeze: idx %d\n", map_idx); in bpf_gen__map_freeze()
1116 map_freeze_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_freeze()
1117 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4, in bpf_gen__map_freeze()
1118 blob_fd_array_off(gen, map_idx)); in bpf_gen__map_freeze()
1120 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); in bpf_gen__map_freeze()
1121 debug_ret(gen, "map_freeze"); in bpf_gen__map_freeze()
1122 emit_check_err(gen); in bpf_gen__map_freeze()