Lines Matching defs:s
228 static inline bool kmem_cache_debug(struct kmem_cache *s) in kmem_cache_debug()
233 static inline bool slub_debug_orig_size(struct kmem_cache *s) in slub_debug_orig_size()
239 void *fixup_red_left(struct kmem_cache *s, void *p) in fixup_red_left()
247 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) in kmem_cache_has_cpu_partial()
338 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } in sysfs_slab_add()
339 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) in sysfs_slab_alias()
346 static inline void debugfs_slab_add(struct kmem_cache *s) { } in debugfs_slab_add()
403 static inline void stat(const struct kmem_cache *s, enum stat_item si) in stat()
415 void stat_add(const struct kmem_cache *s, enum stat_item si, int v) in stat_add()
436 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) in get_node()
473 static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s, in freelist_ptr_encode()
486 static inline void *freelist_ptr_decode(const struct kmem_cache *s, in freelist_ptr_decode()
499 static inline void *get_freepointer(struct kmem_cache *s, void *object) in get_freepointer()
511 static void prefetch_freepointer(const struct kmem_cache *s, void *object) in prefetch_freepointer()
528 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) in get_freepointer_safe()
542 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) in set_freepointer()
557 static inline bool freeptr_outside_object(struct kmem_cache *s) in freeptr_outside_object()
566 static inline unsigned int get_info_end(struct kmem_cache *s) in get_info_end()
606 static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) in slub_set_cpu_partial()
622 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) in slub_get_cpu_partial()
628 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) in slub_set_cpu_partial()
632 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) in slub_get_cpu_partial()
692 static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab, in __slab_update_freelist()
722 static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab, in slab_update_freelist()
759 static inline void set_orig_size(struct kmem_cache *s, in set_orig_size()
784 static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) in get_orig_size()
801 static void __fill_map(unsigned long *obj_map, struct kmem_cache *s, in __fill_map()
848 static inline unsigned int size_from_object(struct kmem_cache *s) in size_from_object()
856 static inline void *restore_red_left(struct kmem_cache *s, void *p) in restore_red_left()
899 static inline int check_valid_pointer(struct kmem_cache *s, in check_valid_pointer()
927 static struct track *get_track(struct kmem_cache *s, void *object, in get_track()
956 static void set_track_update(struct kmem_cache *s, void *object, in set_track_update()
971 static __always_inline void set_track(struct kmem_cache *s, void *object, in set_track()
979 static void init_tracking(struct kmem_cache *s, void *object) in init_tracking()
990 static void print_track(const char *s, struct track *t, unsigned long pr_time) in print_track()
1008 void print_tracking(struct kmem_cache *s, void *object) in print_tracking()
1025 void skip_orig_size_check(struct kmem_cache *s, const void *object) in skip_orig_size_check()
1030 static void slab_bug(struct kmem_cache *s, char *fmt, ...) in slab_bug()
1045 static void slab_fix(struct kmem_cache *s, char *fmt, ...) in slab_fix()
1060 static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) in print_trailer()
1102 static void object_err(struct kmem_cache *s, struct slab *slab, in object_err()
1113 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, in freelist_corrupted()
1127 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab, in slab_err()
1145 static void init_object(struct kmem_cache *s, void *object, u8 val) in init_object()
1179 static void restore_bytes(struct kmem_cache *s, char *message, u8 data, in restore_bytes()
1193 check_bytes_and_report(struct kmem_cache *s, struct slab *slab, in check_bytes_and_report()
1263 static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) in check_pad_bytes()
1286 slab_pad_check(struct kmem_cache *s, struct slab *slab) in slab_pad_check()
1321 static int check_object(struct kmem_cache *s, struct slab *slab, in check_object()
1406 static int check_slab(struct kmem_cache *s, struct slab *slab) in check_slab()
1435 static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search) in on_freelist()
1484 static void trace(struct kmem_cache *s, struct slab *slab, void *object, in trace()
1505 static void add_full(struct kmem_cache *s, in add_full()
1515 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) in remove_full()
1529 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) in inc_slabs_node()
1536 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) in dec_slabs_node()
1545 static void setup_object_debug(struct kmem_cache *s, void *object) in setup_object_debug()
1555 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) in setup_slab_debug()
1565 static inline int alloc_consistency_checks(struct kmem_cache *s, in alloc_consistency_checks()
1582 static noinline bool alloc_debug_processing(struct kmem_cache *s, in alloc_debug_processing()
1610 static inline int free_consistency_checks(struct kmem_cache *s, in free_consistency_checks()
1855 static inline void setup_object_debug(struct kmem_cache *s, void *object) {} in setup_object_debug()
1857 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {} in setup_slab_debug()
1859 static inline bool alloc_debug_processing(struct kmem_cache *s, in alloc_debug_processing()
1862 static inline bool free_debug_processing(struct kmem_cache *s, in free_debug_processing()
1866 static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {} in slab_pad_check()
1867 static inline int check_object(struct kmem_cache *s, struct slab *slab, in check_object()
1870 static inline void set_track(struct kmem_cache *s, void *object, in set_track()
1872 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, in add_full()
1874 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, in remove_full()
1886 static inline void inc_slabs_node(struct kmem_cache *s, int node, in inc_slabs_node()
1888 static inline void dec_slabs_node(struct kmem_cache *s, int node, in dec_slabs_node()
1891 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, in freelist_corrupted()
1957 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, in alloc_slab_obj_exts()
2041 static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, in alloc_slab_obj_exts()
2061 prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) in prepare_slab_obj_exts_hook()
2085 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) in alloc_tagging_slab_alloc_hook()
2102 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, in alloc_tagging_slab_free_hook()
2129 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) in alloc_tagging_slab_alloc_hook()
2134 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, in alloc_tagging_slab_free_hook()
2147 bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, in memcg_slab_post_alloc_hook()
2170 void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, in memcg_slab_free_hook()
2189 struct kmem_cache *s; in memcg_slab_post_charge() local
2224 static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s, in memcg_slab_post_alloc_hook()
2232 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, in memcg_slab_free_hook()
2261 bool slab_free_hook(struct kmem_cache *s, void *x, bool init, in slab_free_hook()
2346 bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail, in slab_free_freelist_hook()
2389 static void *setup_object(struct kmem_cache *s, void *object) in setup_object()
2431 static int init_cache_random_seq(struct kmem_cache *s) in init_cache_random_seq()
2460 struct kmem_cache *s; in init_freelist_randomization() local
2471 static void *next_freelist_entry(struct kmem_cache *s, in next_freelist_entry()
2493 static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) in shuffle_freelist()
2526 static inline int init_cache_random_seq(struct kmem_cache *s) in init_cache_random_seq()
2531 static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) in shuffle_freelist()
2538 struct kmem_cache *s, gfp_t gfp) in account_slab()
2548 struct kmem_cache *s) in unaccount_slab()
2557 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) in allocate_slab()
2624 static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) in new_slab()
2635 static void __free_slab(struct kmem_cache *s, struct slab *slab) in __free_slab()
2658 static void free_slab(struct kmem_cache *s, struct slab *slab) in free_slab()
2674 static void discard_slab(struct kmem_cache *s, struct slab *slab) in discard_slab()
2735 static void *alloc_single_from_partial(struct kmem_cache *s, in alloc_single_from_partial()
2764 static void *alloc_single_from_new_slab(struct kmem_cache *s, in alloc_single_from_new_slab()
2801 static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab, in put_cpu_partial()
2809 static struct slab *get_partial_node(struct kmem_cache *s, in get_partial_node()
2867 static struct slab *get_any_partial(struct kmem_cache *s, in get_any_partial()
2931 static struct slab *get_partial(struct kmem_cache *s, int node, in get_partial()
2987 const struct kmem_cache *s, unsigned long tid) in note_cmpxchg_failure()
3010 static void init_kmem_cache_cpus(struct kmem_cache *s) in init_kmem_cache_cpus()
3028 static void deactivate_slab(struct kmem_cache *s, struct slab *slab, in deactivate_slab()
3109 static void __put_partials(struct kmem_cache *s, struct slab *partial_slab) in __put_partials()
3153 static void put_partials(struct kmem_cache *s) in put_partials()
3167 static void put_partials_cpu(struct kmem_cache *s, in put_partials_cpu()
3185 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) in put_cpu_partial()
3227 static inline void put_partials(struct kmem_cache *s) { } in put_partials()
3228 static inline void put_partials_cpu(struct kmem_cache *s, in put_partials_cpu()
3233 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) in flush_slab()
3256 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) in __flush_cpu_slab()
3276 struct kmem_cache *s; member
3287 struct kmem_cache *s; in flush_cpu_slab() local
3302 static bool has_cpu_slab(int cpu, struct kmem_cache *s) in has_cpu_slab()
3312 static void flush_all_cpus_locked(struct kmem_cache *s) in flush_all_cpus_locked()
3342 static void flush_all(struct kmem_cache *s) in flush_all()
3355 struct kmem_cache *s; in slub_cpu_dead() local
3365 static inline void flush_all_cpus_locked(struct kmem_cache *s) { } in flush_all_cpus_locked()
3366 static inline void flush_all(struct kmem_cache *s) { } in flush_all()
3367 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { } in __flush_cpu_slab()
3396 static inline bool free_debug_processing(struct kmem_cache *s, in free_debug_processing()
3509 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) in slab_out_of_memory()
3545 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { } in slab_out_of_memory()
3558 __update_cpu_freelist_fast(struct kmem_cache *s, in __update_cpu_freelist_fast()
3577 static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) in get_freelist()
3605 static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab) in freeze_slab()
3648 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, in ___slab_alloc()
3894 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, in __slab_alloc()
3915 static __always_inline void *__slab_alloc_node(struct kmem_cache *s, in __slab_alloc_node()
3990 static void *__slab_alloc_node(struct kmem_cache *s, in __slab_alloc_node()
4022 static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s, in maybe_wipe_obj_freeptr()
4032 struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) in slab_pre_alloc_hook()
4045 bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, in slab_post_alloc_hook()
4108 static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru, in slab_alloc_node()
4139 void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags) in kmem_cache_alloc_noprof()
4150 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, in kmem_cache_alloc_lru_noprof()
4184 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node) in kmem_cache_alloc_node_noprof()
4248 struct kmem_cache *s; in __do_kmalloc_node() local
4288 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) in __kmalloc_cache_noprof()
4300 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, in __kmalloc_cache_node_noprof()
4313 struct kmem_cache *s, struct slab *slab, in free_to_partial_list()
4381 static void __slab_free(struct kmem_cache *s, struct slab *slab, in __slab_free()
4509 static __always_inline void do_slab_free(struct kmem_cache *s, in do_slab_free()
4564 static void do_slab_free(struct kmem_cache *s, in do_slab_free()
4573 void slab_free(struct kmem_cache *s, struct slab *slab, void *object, in slab_free()
4586 void memcg_alloc_abort_single(struct kmem_cache *s, void *object) in memcg_alloc_abort_single()
4594 void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head, in slab_free_bulk()
4614 struct kmem_cache *s; in slab_free_after_rcu_debug() local
4651 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) in cache_from_obj()
4675 void kmem_cache_free(struct kmem_cache *s, void *x) in kmem_cache_free()
4711 struct kmem_cache *s; in kfree() local
4736 struct kmem_cache *s; member
4752 int build_detached_freelist(struct kmem_cache *s, size_t size, in build_detached_freelist()
4814 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) in __kmem_cache_free_bulk()
4835 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) in kmem_cache_free_bulk()
4855 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, in __kmem_cache_alloc_bulk()
4925 static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, in __kmem_cache_alloc_bulk()
4955 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, in kmem_cache_alloc_bulk_noprof()
5130 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) in alloc_kmem_cache_cpus()
5151 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) in alloc_kmem_cache_cpus()
5202 static void free_kmem_cache_nodes(struct kmem_cache *s) in free_kmem_cache_nodes()
5213 void __kmem_cache_release(struct kmem_cache *s) in __kmem_cache_release()
5222 static int init_kmem_cache_nodes(struct kmem_cache *s) in init_kmem_cache_nodes()
5247 static void set_cpu_partial(struct kmem_cache *s) in set_cpu_partial()
5284 static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) in calculate_sizes()
5423 static void list_slab_objects(struct kmem_cache *s, struct slab *slab, in list_slab_objects()
5453 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) in free_partial()
5475 bool __kmem_cache_empty(struct kmem_cache *s) in __kmem_cache_empty()
5489 int __kmem_cache_shutdown(struct kmem_cache *s) in __kmem_cache_shutdown()
5512 struct kmem_cache *s = slab->slab_cache; in __kmem_obj_info() local
5617 struct kmem_cache *s; in __check_heap_object() local
5666 static int __kmem_cache_do_shrink(struct kmem_cache *s) in __kmem_cache_do_shrink()
5729 int __kmem_cache_shrink(struct kmem_cache *s) in __kmem_cache_shrink()
5737 struct kmem_cache *s; in slab_mem_going_offline_callback() local
5776 struct kmem_cache *s; in slab_mem_going_online_callback() local
5864 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); in bootstrap() local
5958 struct kmem_cache *s; in __kmem_cache_alias() local
5978 int do_kmem_cache_create(struct kmem_cache *s, const char *name, in do_kmem_cache_create()
6077 static void validate_slab(struct kmem_cache *s, struct slab *slab, in validate_slab()
6097 static int validate_slab_node(struct kmem_cache *s, in validate_slab_node()
6134 long validate_slab_cache(struct kmem_cache *s) in validate_slab_cache()
6211 static int add_location(struct loc_track *t, struct kmem_cache *s, in add_location()
6303 static void process_slab(struct loc_track *t, struct kmem_cache *s, in process_slab()
6337 static ssize_t show_slab_objects(struct kmem_cache *s, in show_slab_objects()
6464 static ssize_t slab_size_show(struct kmem_cache *s, char *buf) in slab_size_show()
6470 static ssize_t align_show(struct kmem_cache *s, char *buf) in align_show()
6476 static ssize_t object_size_show(struct kmem_cache *s, char *buf) in object_size_show()
6482 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) in objs_per_slab_show()
6488 static ssize_t order_show(struct kmem_cache *s, char *buf) in order_show()
6494 static ssize_t min_partial_show(struct kmem_cache *s, char *buf) in min_partial_show()
6499 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, in min_partial_store()
6514 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) in cpu_partial_show()
6524 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, in cpu_partial_store()
6542 static ssize_t ctor_show(struct kmem_cache *s, char *buf) in ctor_show()
6550 static ssize_t aliases_show(struct kmem_cache *s, char *buf) in aliases_show()
6556 static ssize_t partial_show(struct kmem_cache *s, char *buf) in partial_show()
6562 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) in cpu_slabs_show()
6568 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) in objects_partial_show()
6574 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) in slabs_cpu_partial_show()
6615 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) in reclaim_account_show()
6621 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) in hwcache_align_show()
6628 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) in cache_dma_show()
6636 static ssize_t usersize_show(struct kmem_cache *s, char *buf) in usersize_show()
6643 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) in destroy_by_rcu_show()
6650 static ssize_t slabs_show(struct kmem_cache *s, char *buf) in slabs_show()
6656 static ssize_t total_objects_show(struct kmem_cache *s, char *buf) in total_objects_show()
6662 static ssize_t objects_show(struct kmem_cache *s, char *buf) in objects_show()
6668 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) in sanity_checks_show()
6674 static ssize_t trace_show(struct kmem_cache *s, char *buf) in trace_show()
6680 static ssize_t red_zone_show(struct kmem_cache *s, char *buf) in red_zone_show()
6687 static ssize_t poison_show(struct kmem_cache *s, char *buf) in poison_show()
6694 static ssize_t store_user_show(struct kmem_cache *s, char *buf) in store_user_show()
6701 static ssize_t validate_show(struct kmem_cache *s, char *buf) in validate_show()
6706 static ssize_t validate_store(struct kmem_cache *s, in validate_store()
6723 static ssize_t failslab_show(struct kmem_cache *s, char *buf) in failslab_show()
6728 static ssize_t failslab_store(struct kmem_cache *s, const char *buf, in failslab_store()
6744 static ssize_t shrink_show(struct kmem_cache *s, char *buf) in shrink_show()
6749 static ssize_t shrink_store(struct kmem_cache *s, in shrink_store()
6761 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) in remote_node_defrag_ratio_show()
6766 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, in remote_node_defrag_ratio_store()
6786 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) in show_stat()
6818 static void clear_stat(struct kmem_cache *s, enum stat_item si) in clear_stat()
6870 static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf) in skip_kfence_show()
6875 static ssize_t skip_kfence_store(struct kmem_cache *s, in skip_kfence_store()
6977 struct kmem_cache *s; in slab_attr_show() local
6993 struct kmem_cache *s; in slab_attr_store() local
7021 static inline struct kset *cache_kset(struct kmem_cache *s) in cache_kset()
7032 static char *create_unique_id(struct kmem_cache *s) in create_unique_id()
7070 static int sysfs_slab_add(struct kmem_cache *s) in sysfs_slab_add()
7121 void sysfs_slab_unlink(struct kmem_cache *s) in sysfs_slab_unlink()
7126 void sysfs_slab_release(struct kmem_cache *s) in sysfs_slab_release()
7136 struct kmem_cache *s; member
7143 static int sysfs_slab_alias(struct kmem_cache *s, const char *name) in sysfs_slab_alias()
7169 struct kmem_cache *s; in slab_sysfs_init() local
7323 struct kmem_cache *s = file_inode(filep)->i_private; in slab_debug_trace_open() local
7385 static void debugfs_slab_add(struct kmem_cache *s) in debugfs_slab_add()
7401 void debugfs_slab_release(struct kmem_cache *s) in debugfs_slab_release()
7408 struct kmem_cache *s; in slab_debugfs_init() local
7425 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) in get_slabinfo()