1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2022 Google */ 3 #include <linux/bpf.h> 4 #include <linux/btf_ids.h> 5 #include <linux/cgroup.h> 6 #include <linux/kernel.h> 7 #include <linux/seq_file.h> 8 9 #include "../cgroup/cgroup-internal.h" /* cgroup_mutex and cgroup_is_dead */ 10 11 /* cgroup_iter provides four modes of traversal to the cgroup hierarchy. 12 * 13 * 1. Walk the descendants of a cgroup in pre-order. 14 * 2. Walk the descendants of a cgroup in post-order. 15 * 3. Walk the ancestors of a cgroup. 16 * 4. Show the given cgroup only. 17 * 18 * For walking descendants, cgroup_iter can walk in either pre-order or 19 * post-order. For walking ancestors, the iter walks up from a cgroup to 20 * the root. 21 * 22 * The iter program can terminate the walk early by returning 1. Walk 23 * continues if prog returns 0. 24 * 25 * The prog can check (seq->num == 0) to determine whether this is 26 * the first element. The prog may also be passed a NULL cgroup, 27 * which means the walk has completed and the prog has a chance to 28 * do post-processing, such as outputting an epilogue. 29 * 30 * Note: the iter_prog is called with cgroup_mutex held. 31 * 32 * Currently only one session is supported, which means, depending on the 33 * volume of data bpf program intends to send to user space, the number 34 * of cgroups that can be walked is limited. For example, given the current 35 * buffer size is 8 * PAGE_SIZE, if the program sends 64B data for each 36 * cgroup, assuming PAGE_SIZE is 4kb, the total number of cgroups that can 37 * be walked is 512. This is a limitation of cgroup_iter. If the output data 38 * is larger than the kernel buffer size, after all data in the kernel buffer 39 * is consumed by user space, the subsequent read() syscall will signal 40 * EOPNOTSUPP. In order to work around, the user may have to update their 41 * program to reduce the volume of data sent to output. For example, skip 42 * some uninteresting cgroups. 43 */ 44 45 struct bpf_iter__cgroup { 46 __bpf_md_ptr(struct bpf_iter_meta *, meta); 47 __bpf_md_ptr(struct cgroup *, cgroup); 48 }; 49 50 struct cgroup_iter_priv { 51 struct cgroup_subsys_state *start_css; 52 bool visited_all; 53 bool terminate; 54 int order; 55 }; 56 cgroup_iter_seq_start(struct seq_file * seq,loff_t * pos)57 static void *cgroup_iter_seq_start(struct seq_file *seq, loff_t *pos) 58 { 59 struct cgroup_iter_priv *p = seq->private; 60 61 cgroup_lock(); 62 63 /* cgroup_iter doesn't support read across multiple sessions. */ 64 if (*pos > 0) { 65 if (p->visited_all) 66 return NULL; 67 68 /* Haven't visited all, but because cgroup_mutex has dropped, 69 * return -EOPNOTSUPP to indicate incomplete iteration. 70 */ 71 return ERR_PTR(-EOPNOTSUPP); 72 } 73 74 ++*pos; 75 p->terminate = false; 76 p->visited_all = false; 77 if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) 78 return css_next_descendant_pre(NULL, p->start_css); 79 else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) 80 return css_next_descendant_post(NULL, p->start_css); 81 else /* BPF_CGROUP_ITER_SELF_ONLY and BPF_CGROUP_ITER_ANCESTORS_UP */ 82 return p->start_css; 83 } 84 85 static int __cgroup_iter_seq_show(struct seq_file *seq, 86 struct cgroup_subsys_state *css, int in_stop); 87 cgroup_iter_seq_stop(struct seq_file * seq,void * v)88 static void cgroup_iter_seq_stop(struct seq_file *seq, void *v) 89 { 90 struct cgroup_iter_priv *p = seq->private; 91 92 cgroup_unlock(); 93 94 /* pass NULL to the prog for post-processing */ 95 if (!v) { 96 __cgroup_iter_seq_show(seq, NULL, true); 97 p->visited_all = true; 98 } 99 } 100 cgroup_iter_seq_next(struct seq_file * seq,void * v,loff_t * pos)101 static void *cgroup_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) 102 { 103 struct cgroup_subsys_state *curr = (struct cgroup_subsys_state *)v; 104 struct cgroup_iter_priv *p = seq->private; 105 106 ++*pos; 107 if (p->terminate) 108 return NULL; 109 110 if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) 111 return css_next_descendant_pre(curr, p->start_css); 112 else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) 113 return css_next_descendant_post(curr, p->start_css); 114 else if (p->order == BPF_CGROUP_ITER_ANCESTORS_UP) 115 return curr->parent; 116 else /* BPF_CGROUP_ITER_SELF_ONLY */ 117 return NULL; 118 } 119 __cgroup_iter_seq_show(struct seq_file * seq,struct cgroup_subsys_state * css,int in_stop)120 static int __cgroup_iter_seq_show(struct seq_file *seq, 121 struct cgroup_subsys_state *css, int in_stop) 122 { 123 struct cgroup_iter_priv *p = seq->private; 124 struct bpf_iter__cgroup ctx; 125 struct bpf_iter_meta meta; 126 struct bpf_prog *prog; 127 int ret = 0; 128 129 /* cgroup is dead, skip this element */ 130 if (css && cgroup_is_dead(css->cgroup)) 131 return 0; 132 133 ctx.meta = &meta; 134 ctx.cgroup = css ? css->cgroup : NULL; 135 meta.seq = seq; 136 prog = bpf_iter_get_info(&meta, in_stop); 137 if (prog) 138 ret = bpf_iter_run_prog(prog, &ctx); 139 140 /* if prog returns > 0, terminate after this element. */ 141 if (ret != 0) 142 p->terminate = true; 143 144 return 0; 145 } 146 cgroup_iter_seq_show(struct seq_file * seq,void * v)147 static int cgroup_iter_seq_show(struct seq_file *seq, void *v) 148 { 149 return __cgroup_iter_seq_show(seq, (struct cgroup_subsys_state *)v, 150 false); 151 } 152 153 static const struct seq_operations cgroup_iter_seq_ops = { 154 .start = cgroup_iter_seq_start, 155 .next = cgroup_iter_seq_next, 156 .stop = cgroup_iter_seq_stop, 157 .show = cgroup_iter_seq_show, 158 }; 159 BTF_ID_LIST_GLOBAL_SINGLE(bpf_cgroup_btf_id,struct,cgroup)160 BTF_ID_LIST_GLOBAL_SINGLE(bpf_cgroup_btf_id, struct, cgroup) 161 162 static int cgroup_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux) 163 { 164 struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; 165 struct cgroup *cgrp = aux->cgroup.start; 166 167 /* bpf_iter_attach_cgroup() has already acquired an extra reference 168 * for the start cgroup, but the reference may be released after 169 * cgroup_iter_seq_init(), so acquire another reference for the 170 * start cgroup. 171 */ 172 p->start_css = &cgrp->self; 173 css_get(p->start_css); 174 p->terminate = false; 175 p->visited_all = false; 176 p->order = aux->cgroup.order; 177 return 0; 178 } 179 cgroup_iter_seq_fini(void * priv)180 static void cgroup_iter_seq_fini(void *priv) 181 { 182 struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; 183 184 css_put(p->start_css); 185 } 186 187 static const struct bpf_iter_seq_info cgroup_iter_seq_info = { 188 .seq_ops = &cgroup_iter_seq_ops, 189 .init_seq_private = cgroup_iter_seq_init, 190 .fini_seq_private = cgroup_iter_seq_fini, 191 .seq_priv_size = sizeof(struct cgroup_iter_priv), 192 }; 193 bpf_iter_attach_cgroup(struct bpf_prog * prog,union bpf_iter_link_info * linfo,struct bpf_iter_aux_info * aux)194 static int bpf_iter_attach_cgroup(struct bpf_prog *prog, 195 union bpf_iter_link_info *linfo, 196 struct bpf_iter_aux_info *aux) 197 { 198 int fd = linfo->cgroup.cgroup_fd; 199 u64 id = linfo->cgroup.cgroup_id; 200 int order = linfo->cgroup.order; 201 struct cgroup *cgrp; 202 203 if (order != BPF_CGROUP_ITER_DESCENDANTS_PRE && 204 order != BPF_CGROUP_ITER_DESCENDANTS_POST && 205 order != BPF_CGROUP_ITER_ANCESTORS_UP && 206 order != BPF_CGROUP_ITER_SELF_ONLY) 207 return -EINVAL; 208 209 if (fd && id) 210 return -EINVAL; 211 212 if (fd) 213 cgrp = cgroup_v1v2_get_from_fd(fd); 214 else if (id) 215 cgrp = cgroup_get_from_id(id); 216 else /* walk the entire hierarchy by default. */ 217 cgrp = cgroup_get_from_path("/"); 218 219 if (IS_ERR(cgrp)) 220 return PTR_ERR(cgrp); 221 222 aux->cgroup.start = cgrp; 223 aux->cgroup.order = order; 224 return 0; 225 } 226 bpf_iter_detach_cgroup(struct bpf_iter_aux_info * aux)227 static void bpf_iter_detach_cgroup(struct bpf_iter_aux_info *aux) 228 { 229 cgroup_put(aux->cgroup.start); 230 } 231 bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info * aux,struct seq_file * seq)232 static void bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info *aux, 233 struct seq_file *seq) 234 { 235 char *buf; 236 237 buf = kzalloc(PATH_MAX, GFP_KERNEL); 238 if (!buf) { 239 seq_puts(seq, "cgroup_path:\t<unknown>\n"); 240 goto show_order; 241 } 242 243 /* If cgroup_path_ns() fails, buf will be an empty string, cgroup_path 244 * will print nothing. 245 * 246 * Path is in the calling process's cgroup namespace. 247 */ 248 cgroup_path_ns(aux->cgroup.start, buf, PATH_MAX, 249 current->nsproxy->cgroup_ns); 250 seq_printf(seq, "cgroup_path:\t%s\n", buf); 251 kfree(buf); 252 253 show_order: 254 if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_PRE) 255 seq_puts(seq, "order: descendants_pre\n"); 256 else if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_POST) 257 seq_puts(seq, "order: descendants_post\n"); 258 else if (aux->cgroup.order == BPF_CGROUP_ITER_ANCESTORS_UP) 259 seq_puts(seq, "order: ancestors_up\n"); 260 else /* BPF_CGROUP_ITER_SELF_ONLY */ 261 seq_puts(seq, "order: self_only\n"); 262 } 263 bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info * aux,struct bpf_link_info * info)264 static int bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info *aux, 265 struct bpf_link_info *info) 266 { 267 info->iter.cgroup.order = aux->cgroup.order; 268 info->iter.cgroup.cgroup_id = cgroup_id(aux->cgroup.start); 269 return 0; 270 } 271 272 DEFINE_BPF_ITER_FUNC(cgroup, struct bpf_iter_meta *meta, 273 struct cgroup *cgroup) 274 275 static struct bpf_iter_reg bpf_cgroup_reg_info = { 276 .target = "cgroup", 277 .feature = BPF_ITER_RESCHED, 278 .attach_target = bpf_iter_attach_cgroup, 279 .detach_target = bpf_iter_detach_cgroup, 280 .show_fdinfo = bpf_iter_cgroup_show_fdinfo, 281 .fill_link_info = bpf_iter_cgroup_fill_link_info, 282 .ctx_arg_info_size = 1, 283 .ctx_arg_info = { 284 { offsetof(struct bpf_iter__cgroup, cgroup), 285 PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, 286 }, 287 .seq_info = &cgroup_iter_seq_info, 288 }; 289 bpf_cgroup_iter_init(void)290 static int __init bpf_cgroup_iter_init(void) 291 { 292 bpf_cgroup_reg_info.ctx_arg_info[0].btf_id = bpf_cgroup_btf_id[0]; 293 return bpf_iter_reg_target(&bpf_cgroup_reg_info); 294 } 295 296 late_initcall(bpf_cgroup_iter_init); 297 298 struct bpf_iter_css { 299 __u64 __opaque[3]; 300 } __attribute__((aligned(8))); 301 302 struct bpf_iter_css_kern { 303 struct cgroup_subsys_state *start; 304 struct cgroup_subsys_state *pos; 305 unsigned int flags; 306 } __attribute__((aligned(8))); 307 308 __bpf_kfunc_start_defs(); 309 bpf_iter_css_new(struct bpf_iter_css * it,struct cgroup_subsys_state * start,unsigned int flags)310 __bpf_kfunc int bpf_iter_css_new(struct bpf_iter_css *it, 311 struct cgroup_subsys_state *start, unsigned int flags) 312 { 313 struct bpf_iter_css_kern *kit = (void *)it; 314 315 BUILD_BUG_ON(sizeof(struct bpf_iter_css_kern) > sizeof(struct bpf_iter_css)); 316 BUILD_BUG_ON(__alignof__(struct bpf_iter_css_kern) != __alignof__(struct bpf_iter_css)); 317 318 kit->start = NULL; 319 switch (flags) { 320 case BPF_CGROUP_ITER_DESCENDANTS_PRE: 321 case BPF_CGROUP_ITER_DESCENDANTS_POST: 322 case BPF_CGROUP_ITER_ANCESTORS_UP: 323 break; 324 default: 325 return -EINVAL; 326 } 327 328 kit->start = start; 329 kit->pos = NULL; 330 kit->flags = flags; 331 return 0; 332 } 333 bpf_iter_css_next(struct bpf_iter_css * it)334 __bpf_kfunc struct cgroup_subsys_state *bpf_iter_css_next(struct bpf_iter_css *it) 335 { 336 struct bpf_iter_css_kern *kit = (void *)it; 337 338 if (!kit->start) 339 return NULL; 340 341 switch (kit->flags) { 342 case BPF_CGROUP_ITER_DESCENDANTS_PRE: 343 kit->pos = css_next_descendant_pre(kit->pos, kit->start); 344 break; 345 case BPF_CGROUP_ITER_DESCENDANTS_POST: 346 kit->pos = css_next_descendant_post(kit->pos, kit->start); 347 break; 348 case BPF_CGROUP_ITER_ANCESTORS_UP: 349 kit->pos = kit->pos ? kit->pos->parent : kit->start; 350 } 351 352 return kit->pos; 353 } 354 bpf_iter_css_destroy(struct bpf_iter_css * it)355 __bpf_kfunc void bpf_iter_css_destroy(struct bpf_iter_css *it) 356 { 357 } 358 359 __bpf_kfunc_end_defs(); 360