1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2019 Intel Corporation. */ 3 4 #include <linux/hash.h> 5 #include <linux/bpf.h> 6 #include <linux/filter.h> 7 #include <linux/static_call.h> 8 9 /* The BPF dispatcher is a multiway branch code generator. The 10 * dispatcher is a mechanism to avoid the performance penalty of an 11 * indirect call, which is expensive when retpolines are enabled. A 12 * dispatch client registers a BPF program into the dispatcher, and if 13 * there is available room in the dispatcher a direct call to the BPF 14 * program will be generated. All calls to the BPF programs called via 15 * the dispatcher will then be a direct call, instead of an 16 * indirect. The dispatcher hijacks a trampoline function it via the 17 * __fentry__ of the trampoline. The trampoline function has the 18 * following signature: 19 * 20 * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi, 21 * unsigned int (*bpf_func)(const void *, 22 * const struct bpf_insn *)); 23 */ 24 bpf_dispatcher_find_prog(struct bpf_dispatcher * d,struct bpf_prog * prog)25 static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog( 26 struct bpf_dispatcher *d, struct bpf_prog *prog) 27 { 28 int i; 29 30 for (i = 0; i < BPF_DISPATCHER_MAX; i++) { 31 if (prog == d->progs[i].prog) 32 return &d->progs[i]; 33 } 34 return NULL; 35 } 36 bpf_dispatcher_find_free(struct bpf_dispatcher * d)37 static struct bpf_dispatcher_prog *bpf_dispatcher_find_free( 38 struct bpf_dispatcher *d) 39 { 40 return bpf_dispatcher_find_prog(d, NULL); 41 } 42 bpf_dispatcher_add_prog(struct bpf_dispatcher * d,struct bpf_prog * prog)43 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d, 44 struct bpf_prog *prog) 45 { 46 struct bpf_dispatcher_prog *entry; 47 48 if (!prog) 49 return false; 50 51 entry = bpf_dispatcher_find_prog(d, prog); 52 if (entry) { 53 refcount_inc(&entry->users); 54 return false; 55 } 56 57 entry = bpf_dispatcher_find_free(d); 58 if (!entry) 59 return false; 60 61 bpf_prog_inc(prog); 62 entry->prog = prog; 63 refcount_set(&entry->users, 1); 64 d->num_progs++; 65 return true; 66 } 67 bpf_dispatcher_remove_prog(struct bpf_dispatcher * d,struct bpf_prog * prog)68 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d, 69 struct bpf_prog *prog) 70 { 71 struct bpf_dispatcher_prog *entry; 72 73 if (!prog) 74 return false; 75 76 entry = bpf_dispatcher_find_prog(d, prog); 77 if (!entry) 78 return false; 79 80 if (refcount_dec_and_test(&entry->users)) { 81 entry->prog = NULL; 82 bpf_prog_put(prog); 83 d->num_progs--; 84 return true; 85 } 86 return false; 87 } 88 arch_prepare_bpf_dispatcher(void * image,void * buf,s64 * funcs,int num_funcs)89 int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs) 90 { 91 return -ENOTSUPP; 92 } 93 bpf_dispatcher_prepare(struct bpf_dispatcher * d,void * image,void * buf)94 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf) 95 { 96 s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0]; 97 int i; 98 99 for (i = 0; i < BPF_DISPATCHER_MAX; i++) { 100 if (d->progs[i].prog) 101 *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func; 102 } 103 return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs); 104 } 105 bpf_dispatcher_update(struct bpf_dispatcher * d,int prev_num_progs)106 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) 107 { 108 void *new, *tmp; 109 u32 noff = 0; 110 111 if (prev_num_progs) 112 noff = d->image_off ^ (PAGE_SIZE / 2); 113 114 new = d->num_progs ? d->image + noff : NULL; 115 tmp = d->num_progs ? d->rw_image + noff : NULL; 116 if (new) { 117 /* Prepare the dispatcher in d->rw_image. Then use 118 * bpf_arch_text_copy to update d->image, which is RO+X. 119 */ 120 if (bpf_dispatcher_prepare(d, new, tmp)) 121 return; 122 if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2))) 123 return; 124 } 125 126 __BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func); 127 128 /* Make sure all the callers executing the previous/old half of the 129 * image leave it, so following update call can modify it safely. 130 */ 131 synchronize_rcu(); 132 133 if (new) 134 d->image_off = noff; 135 } 136 bpf_dispatcher_change_prog(struct bpf_dispatcher * d,struct bpf_prog * from,struct bpf_prog * to)137 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 138 struct bpf_prog *to) 139 { 140 bool changed = false; 141 int prev_num_progs; 142 143 if (from == to) 144 return; 145 146 mutex_lock(&d->mutex); 147 if (!d->image) { 148 d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); 149 if (!d->image) 150 goto out; 151 d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); 152 if (!d->rw_image) { 153 bpf_prog_pack_free(d->image, PAGE_SIZE); 154 d->image = NULL; 155 goto out; 156 } 157 bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym); 158 } 159 160 prev_num_progs = d->num_progs; 161 changed |= bpf_dispatcher_remove_prog(d, from); 162 changed |= bpf_dispatcher_add_prog(d, to); 163 164 if (!changed) 165 goto out; 166 167 bpf_dispatcher_update(d, prev_num_progs); 168 out: 169 mutex_unlock(&d->mutex); 170 } 171