1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2023 Isovalent */
3 
4 #include <linux/bpf.h>
5 #include <linux/bpf_mprog.h>
6 #include <linux/netdevice.h>
7 
8 #include <net/tcx.h>
9 
tcx_prog_attach(const union bpf_attr * attr,struct bpf_prog * prog)10 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
11 {
12 	bool created, ingress = attr->attach_type == BPF_TCX_INGRESS;
13 	struct net *net = current->nsproxy->net_ns;
14 	struct bpf_mprog_entry *entry, *entry_new;
15 	struct bpf_prog *replace_prog = NULL;
16 	struct net_device *dev;
17 	int ret;
18 
19 	rtnl_lock();
20 	dev = __dev_get_by_index(net, attr->target_ifindex);
21 	if (!dev) {
22 		ret = -ENODEV;
23 		goto out;
24 	}
25 	if (attr->attach_flags & BPF_F_REPLACE) {
26 		replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
27 						 prog->type);
28 		if (IS_ERR(replace_prog)) {
29 			ret = PTR_ERR(replace_prog);
30 			replace_prog = NULL;
31 			goto out;
32 		}
33 	}
34 	entry = tcx_entry_fetch_or_create(dev, ingress, &created);
35 	if (!entry) {
36 		ret = -ENOMEM;
37 		goto out;
38 	}
39 	ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
40 			       attr->attach_flags, attr->relative_fd,
41 			       attr->expected_revision);
42 	if (!ret) {
43 		if (entry != entry_new) {
44 			tcx_entry_update(dev, entry_new, ingress);
45 			tcx_entry_sync();
46 			tcx_skeys_inc(ingress);
47 		}
48 		bpf_mprog_commit(entry);
49 	} else if (created) {
50 		tcx_entry_free(entry);
51 	}
52 out:
53 	if (replace_prog)
54 		bpf_prog_put(replace_prog);
55 	rtnl_unlock();
56 	return ret;
57 }
58 
tcx_prog_detach(const union bpf_attr * attr,struct bpf_prog * prog)59 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
60 {
61 	bool ingress = attr->attach_type == BPF_TCX_INGRESS;
62 	struct net *net = current->nsproxy->net_ns;
63 	struct bpf_mprog_entry *entry, *entry_new;
64 	struct net_device *dev;
65 	int ret;
66 
67 	rtnl_lock();
68 	dev = __dev_get_by_index(net, attr->target_ifindex);
69 	if (!dev) {
70 		ret = -ENODEV;
71 		goto out;
72 	}
73 	entry = tcx_entry_fetch(dev, ingress);
74 	if (!entry) {
75 		ret = -ENOENT;
76 		goto out;
77 	}
78 	ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
79 			       attr->relative_fd, attr->expected_revision);
80 	if (!ret) {
81 		if (!tcx_entry_is_active(entry_new))
82 			entry_new = NULL;
83 		tcx_entry_update(dev, entry_new, ingress);
84 		tcx_entry_sync();
85 		tcx_skeys_dec(ingress);
86 		bpf_mprog_commit(entry);
87 		if (!entry_new)
88 			tcx_entry_free(entry);
89 	}
90 out:
91 	rtnl_unlock();
92 	return ret;
93 }
94 
tcx_uninstall(struct net_device * dev,bool ingress)95 void tcx_uninstall(struct net_device *dev, bool ingress)
96 {
97 	struct bpf_mprog_entry *entry, *entry_new = NULL;
98 	struct bpf_tuple tuple = {};
99 	struct bpf_mprog_fp *fp;
100 	struct bpf_mprog_cp *cp;
101 	bool active;
102 
103 	entry = tcx_entry_fetch(dev, ingress);
104 	if (!entry)
105 		return;
106 	active = tcx_entry(entry)->miniq_active;
107 	if (active)
108 		bpf_mprog_clear_all(entry, &entry_new);
109 	tcx_entry_update(dev, entry_new, ingress);
110 	tcx_entry_sync();
111 	bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
112 		if (tuple.link)
113 			tcx_link(tuple.link)->dev = NULL;
114 		else
115 			bpf_prog_put(tuple.prog);
116 		tcx_skeys_dec(ingress);
117 	}
118 	if (!active)
119 		tcx_entry_free(entry);
120 }
121 
tcx_prog_query(const union bpf_attr * attr,union bpf_attr __user * uattr)122 int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
123 {
124 	bool ingress = attr->query.attach_type == BPF_TCX_INGRESS;
125 	struct net *net = current->nsproxy->net_ns;
126 	struct net_device *dev;
127 	int ret;
128 
129 	rtnl_lock();
130 	dev = __dev_get_by_index(net, attr->query.target_ifindex);
131 	if (!dev) {
132 		ret = -ENODEV;
133 		goto out;
134 	}
135 	ret = bpf_mprog_query(attr, uattr, tcx_entry_fetch(dev, ingress));
136 out:
137 	rtnl_unlock();
138 	return ret;
139 }
140 
tcx_link_prog_attach(struct bpf_link * link,u32 flags,u32 id_or_fd,u64 revision)141 static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd,
142 				u64 revision)
143 {
144 	struct tcx_link *tcx = tcx_link(link);
145 	bool created, ingress = tcx->location == BPF_TCX_INGRESS;
146 	struct bpf_mprog_entry *entry, *entry_new;
147 	struct net_device *dev = tcx->dev;
148 	int ret;
149 
150 	ASSERT_RTNL();
151 	entry = tcx_entry_fetch_or_create(dev, ingress, &created);
152 	if (!entry)
153 		return -ENOMEM;
154 	ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
155 			       id_or_fd, revision);
156 	if (!ret) {
157 		if (entry != entry_new) {
158 			tcx_entry_update(dev, entry_new, ingress);
159 			tcx_entry_sync();
160 			tcx_skeys_inc(ingress);
161 		}
162 		bpf_mprog_commit(entry);
163 	} else if (created) {
164 		tcx_entry_free(entry);
165 	}
166 	return ret;
167 }
168 
tcx_link_release(struct bpf_link * link)169 static void tcx_link_release(struct bpf_link *link)
170 {
171 	struct tcx_link *tcx = tcx_link(link);
172 	bool ingress = tcx->location == BPF_TCX_INGRESS;
173 	struct bpf_mprog_entry *entry, *entry_new;
174 	struct net_device *dev;
175 	int ret = 0;
176 
177 	rtnl_lock();
178 	dev = tcx->dev;
179 	if (!dev)
180 		goto out;
181 	entry = tcx_entry_fetch(dev, ingress);
182 	if (!entry) {
183 		ret = -ENOENT;
184 		goto out;
185 	}
186 	ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
187 	if (!ret) {
188 		if (!tcx_entry_is_active(entry_new))
189 			entry_new = NULL;
190 		tcx_entry_update(dev, entry_new, ingress);
191 		tcx_entry_sync();
192 		tcx_skeys_dec(ingress);
193 		bpf_mprog_commit(entry);
194 		if (!entry_new)
195 			tcx_entry_free(entry);
196 		tcx->dev = NULL;
197 	}
198 out:
199 	WARN_ON_ONCE(ret);
200 	rtnl_unlock();
201 }
202 
tcx_link_update(struct bpf_link * link,struct bpf_prog * nprog,struct bpf_prog * oprog)203 static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog,
204 			   struct bpf_prog *oprog)
205 {
206 	struct tcx_link *tcx = tcx_link(link);
207 	bool ingress = tcx->location == BPF_TCX_INGRESS;
208 	struct bpf_mprog_entry *entry, *entry_new;
209 	struct net_device *dev;
210 	int ret = 0;
211 
212 	rtnl_lock();
213 	dev = tcx->dev;
214 	if (!dev) {
215 		ret = -ENOLINK;
216 		goto out;
217 	}
218 	if (oprog && link->prog != oprog) {
219 		ret = -EPERM;
220 		goto out;
221 	}
222 	oprog = link->prog;
223 	if (oprog == nprog) {
224 		bpf_prog_put(nprog);
225 		goto out;
226 	}
227 	entry = tcx_entry_fetch(dev, ingress);
228 	if (!entry) {
229 		ret = -ENOENT;
230 		goto out;
231 	}
232 	ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
233 			       BPF_F_REPLACE | BPF_F_ID,
234 			       link->prog->aux->id, 0);
235 	if (!ret) {
236 		WARN_ON_ONCE(entry != entry_new);
237 		oprog = xchg(&link->prog, nprog);
238 		bpf_prog_put(oprog);
239 		bpf_mprog_commit(entry);
240 	}
241 out:
242 	rtnl_unlock();
243 	return ret;
244 }
245 
tcx_link_dealloc(struct bpf_link * link)246 static void tcx_link_dealloc(struct bpf_link *link)
247 {
248 	kfree(tcx_link(link));
249 }
250 
tcx_link_fdinfo(const struct bpf_link * link,struct seq_file * seq)251 static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
252 {
253 	const struct tcx_link *tcx = tcx_link(link);
254 	u32 ifindex = 0;
255 
256 	rtnl_lock();
257 	if (tcx->dev)
258 		ifindex = tcx->dev->ifindex;
259 	rtnl_unlock();
260 
261 	seq_printf(seq, "ifindex:\t%u\n", ifindex);
262 	seq_printf(seq, "attach_type:\t%u (%s)\n",
263 		   tcx->location,
264 		   tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress");
265 }
266 
tcx_link_fill_info(const struct bpf_link * link,struct bpf_link_info * info)267 static int tcx_link_fill_info(const struct bpf_link *link,
268 			      struct bpf_link_info *info)
269 {
270 	const struct tcx_link *tcx = tcx_link(link);
271 	u32 ifindex = 0;
272 
273 	rtnl_lock();
274 	if (tcx->dev)
275 		ifindex = tcx->dev->ifindex;
276 	rtnl_unlock();
277 
278 	info->tcx.ifindex = ifindex;
279 	info->tcx.attach_type = tcx->location;
280 	return 0;
281 }
282 
tcx_link_detach(struct bpf_link * link)283 static int tcx_link_detach(struct bpf_link *link)
284 {
285 	tcx_link_release(link);
286 	return 0;
287 }
288 
289 static const struct bpf_link_ops tcx_link_lops = {
290 	.release	= tcx_link_release,
291 	.detach		= tcx_link_detach,
292 	.dealloc	= tcx_link_dealloc,
293 	.update_prog	= tcx_link_update,
294 	.show_fdinfo	= tcx_link_fdinfo,
295 	.fill_link_info	= tcx_link_fill_info,
296 };
297 
tcx_link_init(struct tcx_link * tcx,struct bpf_link_primer * link_primer,const union bpf_attr * attr,struct net_device * dev,struct bpf_prog * prog)298 static int tcx_link_init(struct tcx_link *tcx,
299 			 struct bpf_link_primer *link_primer,
300 			 const union bpf_attr *attr,
301 			 struct net_device *dev,
302 			 struct bpf_prog *prog)
303 {
304 	bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog);
305 	tcx->location = attr->link_create.attach_type;
306 	tcx->dev = dev;
307 	return bpf_link_prime(&tcx->link, link_primer);
308 }
309 
tcx_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)310 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
311 {
312 	struct net *net = current->nsproxy->net_ns;
313 	struct bpf_link_primer link_primer;
314 	struct net_device *dev;
315 	struct tcx_link *tcx;
316 	int ret;
317 
318 	rtnl_lock();
319 	dev = __dev_get_by_index(net, attr->link_create.target_ifindex);
320 	if (!dev) {
321 		ret = -ENODEV;
322 		goto out;
323 	}
324 	tcx = kzalloc(sizeof(*tcx), GFP_USER);
325 	if (!tcx) {
326 		ret = -ENOMEM;
327 		goto out;
328 	}
329 	ret = tcx_link_init(tcx, &link_primer, attr, dev, prog);
330 	if (ret) {
331 		kfree(tcx);
332 		goto out;
333 	}
334 	ret = tcx_link_prog_attach(&tcx->link, attr->link_create.flags,
335 				   attr->link_create.tcx.relative_fd,
336 				   attr->link_create.tcx.expected_revision);
337 	if (ret) {
338 		tcx->dev = NULL;
339 		bpf_link_cleanup(&link_primer);
340 		goto out;
341 	}
342 	ret = bpf_link_settle(&link_primer);
343 out:
344 	rtnl_unlock();
345 	return ret;
346 }
347