1 /* SPDX-License-Identifier: GPL-2.0 */
8  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
9  * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
23  * DEFINE_STATIC_KEY_TRUE(key);
24  * DEFINE_STATIC_KEY_FALSE(key);
31  * self-modifying code. Assuming toolchain and architecture support, if we
32  * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
33  * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
34  * (which defaults to false - and the true block is placed out of line).
35  * Similarly, we can define an initially true key via
36  * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
37  * "if (static_branch_unlikely(&key))", in which case we will generate an
38  * unconditional branch to the out-of-line true branch. Keys that are
42  * At runtime we can change the branch target by setting the key
45  * these calls then we run-time modify the branch target via a
46  * no-op -> jump or jump -> no-op conversion. For example, for an
47  * initially false key that is used in an "if (static_branch_unlikely(&key))"
48  * statement, setting the key to true requires us to patch in a jump
49  * to the out-of-line of true branch.
52  * the key or branch direction via static_branch_{inc,dec}. Thus,
61  * to the out-of-line block.
71  * Additional babbling in: Documentation/staging/static-keys.rst
81 #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized,		      \  argument
82 				    "%s(): static key '%pS' used before call to jump_label_init()", \
83 				    __func__, (key))
96  * bit 0 => 1 if key is initially true
98  * bit 1 => 1 if points to struct static_key_mod
120 	long key;	// key may be far away from the core kernel under KASLR  member
125 	return (unsigned long)&entry->code + entry->code;  in jump_entry_code()
130 	return (unsigned long)&entry->target + entry->target;  in jump_entry_target()
135 	long offset = entry->key & ~3L;  in jump_entry_key()
137 	return (struct static_key *)((unsigned long)&entry->key + offset);  in jump_entry_key()
144 	return entry->code;  in jump_entry_code()
149 	return entry->target;  in jump_entry_target()
154 	return (struct static_key *)((unsigned long)entry->key & ~3UL);  in jump_entry_key()
161 	return (unsigned long)entry->key & 1UL;  in jump_entry_is_branch()
166 	return (unsigned long)entry->key & 2UL;  in jump_entry_is_init()
172 		entry->key |= 2;  in jump_entry_set_init()
174 		entry->key &= ~2;  in jump_entry_set_init()
201 #define JUMP_TYPE_TRUE		1UL
205 static __always_inline bool static_key_false(struct static_key *key)  in static_key_false()  argument
207 	return arch_static_branch(key, false);  in static_key_false()
210 static __always_inline bool static_key_true(struct static_key *key)  in static_key_true()  argument
212 	return !arch_static_branch(key, true);  in static_key_true()
228 extern bool static_key_slow_inc(struct static_key *key);
229 extern bool static_key_fast_inc_not_disabled(struct static_key *key);
230 extern void static_key_slow_dec(struct static_key *key);
231 extern bool static_key_slow_inc_cpuslocked(struct static_key *key);
232 extern void static_key_slow_dec_cpuslocked(struct static_key *key);
233 extern int static_key_count(struct static_key *key);
234 extern void static_key_enable(struct static_key *key);
235 extern void static_key_disable(struct static_key *key);
236 extern void static_key_enable_cpuslocked(struct static_key *key);
237 extern void static_key_disable_cpuslocked(struct static_key *key);
243  * in 'low-level' headers. Thus, we are initializing .enabled with a
248 	{ .enabled = { 1 },					\
259 static __always_inline int static_key_count(struct static_key *key)  in static_key_count()  argument
261 	return raw_atomic_read(&key->enabled);  in static_key_count()
271 static __always_inline bool static_key_false(struct static_key *key)  in static_key_false()  argument
273 	if (unlikely_notrace(static_key_count(key) > 0))  in static_key_false()
278 static __always_inline bool static_key_true(struct static_key *key)  in static_key_true()  argument
280 	if (likely_notrace(static_key_count(key) > 0))  in static_key_true()
285 static inline bool static_key_fast_inc_not_disabled(struct static_key *key)  in static_key_fast_inc_not_disabled()  argument
289 	STATIC_KEY_CHECK_USE(key);  in static_key_fast_inc_not_disabled()
291 	 * Prevent key->enabled getting negative to follow the same semantics  in static_key_fast_inc_not_disabled()
294 	v = atomic_read(&key->enabled);  in static_key_fast_inc_not_disabled()
296 		if (v < 0 || (v + 1) < 0)  in static_key_fast_inc_not_disabled()
298 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));  in static_key_fast_inc_not_disabled()
301 #define static_key_slow_inc(key)	static_key_fast_inc_not_disabled(key)  argument
303 static inline void static_key_slow_dec(struct static_key *key)  in static_key_slow_dec()  argument
305 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec()
306 	atomic_dec(&key->enabled);  in static_key_slow_dec()
309 #define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)  argument
310 #define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)  argument
320 static inline void static_key_enable(struct static_key *key)  in static_key_enable()  argument
322 	STATIC_KEY_CHECK_USE(key);  in static_key_enable()
324 	if (atomic_read(&key->enabled) != 0) {  in static_key_enable()
325 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable()
328 	atomic_set(&key->enabled, 1);  in static_key_enable()
331 static inline void static_key_disable(struct static_key *key)  in static_key_disable()  argument
333 	STATIC_KEY_CHECK_USE(key);  in static_key_disable()
335 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable()
336 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable()
339 	atomic_set(&key->enabled, 0);  in static_key_disable()
345 #define STATIC_KEY_INIT_TRUE	{ .enabled = ATOMIC_INIT(1) }
353 /* -------------------------------------------------------------------------- */
363 	struct static_key key;  member
367 	struct static_key key;  member
370 #define STATIC_KEY_TRUE_INIT  (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE,  }
371 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
393 		[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT,	\
398 		[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,	\
434  * type\branch|	likely (1)	      |	unlikely (0)
435  * -----------+-----------------------+------------------
437  *  true (1)  |	   ...		      |	   ...
439  *            |    <br-stmts>	      |	1: ...
442  *            |			      |	L: <br-stmts>
443  *            |			      |	   jmp 1b
445  * -----------+-----------------------+------------------
449  *            |    <br-stmts>	      |	1: ...
452  *            |			      |	L: <br-stmts>
453  *            |			      |	   jmp 1b
455  * -----------+-----------------------+------------------
458  * type: 0 = false, 1 = true.
460  * The branch type is encoded in the LSB of jump_entry::key,
461  * branch: 0 = unlikely, 1 = likely.
466  * -----------------------------+-----------
468  *	0	0	1	| JMP
469  *	0	1	0	| NOP
470  *	0	1	1	| JMP
472  *	1	0	0	| JMP
473  *	1	0	1	| NOP
474  *	1	1	0	| JMP
475  *	1	1	1	| NOP
489 		branch = !arch_static_branch(&(x)->key, true);			\
491 		branch = !arch_static_branch_jump(&(x)->key, true);		\
501 		branch = arch_static_branch_jump(&(x)->key, false);		\
503 		branch = arch_static_branch(&(x)->key, false);			\
511 #define static_branch_likely(x)		likely_notrace(static_key_enabled(&(x)->key))
512 #define static_branch_unlikely(x)	unlikely_notrace(static_key_enabled(&(x)->key))
524 #define static_branch_inc(x)		static_key_slow_inc(&(x)->key)
525 #define static_branch_dec(x)		static_key_slow_dec(&(x)->key)
526 #define static_branch_inc_cpuslocked(x)	static_key_slow_inc_cpuslocked(&(x)->key)
527 #define static_branch_dec_cpuslocked(x)	static_key_slow_dec_cpuslocked(&(x)->key)
533 #define static_branch_enable(x)			static_key_enable(&(x)->key)
534 #define static_branch_disable(x)		static_key_disable(&(x)->key)
535 #define static_branch_enable_cpuslocked(x)	static_key_enable_cpuslocked(&(x)->key)
536 #define static_branch_disable_cpuslocked(x)	static_key_disable_cpuslocked(&(x)->key)