122  * @con:	Console to update
128 static inline void nbcon_state_set(struct console *con, struct nbcon_state *new)  in nbcon_state_set()  argument
130 	atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom);  in nbcon_state_set()
135  * @con:	Console to read
138 static inline void nbcon_state_read(struct console *con, struct nbcon_state *state)  in nbcon_state_read()  argument
140 	state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state));  in nbcon_state_read()
145  * @con:	Console to update
151 static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur,  in nbcon_state_try_cmpxchg()  argument
154 	return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom);  in nbcon_state_try_cmpxchg()
159  * @con:	Console to read the sequence of
161  * Return:	Sequence number of the next record to print on @con.
163 u64 nbcon_seq_read(struct console *con)  in nbcon_seq_read()  argument
165 	unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq));  in nbcon_seq_read()
172  * @con:	Console to work on
178 void nbcon_seq_force(struct console *con, u64 seq)  in nbcon_seq_force()  argument
188 	atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));  in nbcon_seq_force()
197  * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
205 	struct console *con = ctxt->console;  in nbcon_seq_try_update()  local
207 	if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq,  in nbcon_seq_try_update()
211 		ctxt->seq = nbcon_seq_read(con);  in nbcon_seq_try_update()
241 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_direct()  local
273 	} while (!nbcon_state_try_cmpxchg(con, cur, &new));  in nbcon_context_try_acquire_direct()
339 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_requested()  local
369 	if (!nbcon_state_try_cmpxchg(con, cur, &new)) {  in nbcon_context_try_acquire_requested()
422 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_handover()  local
455 	if (!nbcon_state_try_cmpxchg(con, cur, &new))  in nbcon_context_try_acquire_handover()
477 		nbcon_state_read(con, cur);  in nbcon_context_try_acquire_handover()
493 		if (nbcon_state_try_cmpxchg(con, cur, &new)) {  in nbcon_context_try_acquire_handover()
528 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_hostile()  local
552 	} while (!nbcon_state_try_cmpxchg(con, cur, &new));  in nbcon_context_try_acquire_hostile()
563  * Context:	Under @ctxt->con->device_lock() or local_irq_save().
574 	struct console *con = ctxt->console;  in nbcon_context_try_acquire()  local
578 	nbcon_state_read(con, &cur);  in nbcon_context_try_acquire()
601 		ctxt->pbufs = con->pbufs;  in nbcon_context_try_acquire()
654 	struct console *con = ctxt->console;  in nbcon_context_release()  local
658 	nbcon_state_read(con, &cur);  in nbcon_context_release()
673 	} while (!nbcon_state_try_cmpxchg(con, &cur, &new));  in nbcon_context_release()
775 	struct console *con = ctxt->console;  in nbcon_can_proceed()  local
778 	nbcon_state_read(con, &cur);  in nbcon_can_proceed()
788  * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
808 	struct console *con = ctxt->console;  in __nbcon_context_update_unsafe()  local
812 	nbcon_state_read(con, &cur);  in __nbcon_context_update_unsafe()
827 	} while (!nbcon_state_try_cmpxchg(con, &cur, &new));  in __nbcon_context_update_unsafe()
838 	struct console *con = ctxt->console;  in nbcon_write_context_set_buf()  local
843 	nbcon_state_read(con, &cur);  in nbcon_write_context_set_buf()
943 	struct console *con = ctxt->console;  in nbcon_emit_next_record()  local
944 	bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;  in nbcon_emit_next_record()
962 	if (WARN_ON_ONCE((use_atomic && !con->write_atomic) ||  in nbcon_emit_next_record()
963 			 !(console_srcu_read_flags(con) & CON_NBCON))) {  in nbcon_emit_next_record()
982 	 * @con->dropped is not protected in case of an unsafe hostile  in nbcon_emit_next_record()
986 	con_dropped = data_race(READ_ONCE(con->dropped));  in nbcon_emit_next_record()
997 	ulseq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_prev_seq));  in nbcon_emit_next_record()
1007 		nbcon_state_read(con, &cur);  in nbcon_emit_next_record()
1011 		atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_prev_seq), &ulseq,  in nbcon_emit_next_record()
1018 	/* For skipped records just update seq/dropped in @con. */  in nbcon_emit_next_record()
1026 		con->write_atomic(con, wctxt);  in nbcon_emit_next_record()
1028 		con->write_thread(con, wctxt);  in nbcon_emit_next_record()
1062 		WRITE_ONCE(con->dropped, dropped);  in nbcon_emit_next_record()
1090 	struct console *con = ctxt->console;  in nbcon_emit_one()  local
1095 		con->device_lock(con, &flags);  in nbcon_emit_one()
1123 		con->device_unlock(con, flags);  in nbcon_emit_one()
1129  * @con:	Console to operate on
1138 static bool nbcon_kthread_should_wakeup(struct console *con, struct nbcon_context *ctxt)  in nbcon_kthread_should_wakeup()  argument
1149 	flags = console_srcu_read_flags(con);  in nbcon_kthread_should_wakeup()
1150 	if (console_is_usable(con, flags, false)) {  in nbcon_kthread_should_wakeup()
1152 		ctxt->seq = nbcon_seq_read(con);  in nbcon_kthread_should_wakeup()
1169 	struct console *con = __console;  in nbcon_kthread_func()  local
1171 		.ctxt.console	= con,  in nbcon_kthread_func()
1190 	rcuwait_wait_event(&con->rcuwait,  in nbcon_kthread_func()
1191 			   nbcon_kthread_should_wakeup(con, ctxt),  in nbcon_kthread_func()
1207 		con_flags = console_srcu_read_flags(con);  in nbcon_kthread_func()
1209 		if (console_is_usable(con, con_flags, false))  in nbcon_kthread_func()
1227 	struct console *con = container_of(irq_work, struct console, irq_work);  in nbcon_irq_work()  local
1229 	nbcon_kthread_wake(con);  in nbcon_irq_work()
1254 	struct console *con;  in nbcon_kthreads_wake()  local
1261 	for_each_console_srcu(con) {  in nbcon_kthreads_wake()
1262 		if (!(console_srcu_read_flags(con) & CON_NBCON))  in nbcon_kthreads_wake()
1270 		if (rcuwait_has_sleeper(&con->rcuwait))  in nbcon_kthreads_wake()
1271 			irq_work_queue(&con->irq_work);  in nbcon_kthreads_wake()
1278  * @con:	Console to operate on
1280 void nbcon_kthread_stop(struct console *con)  in nbcon_kthread_stop()  argument
1284 	if (!con->kthread)  in nbcon_kthread_stop()
1287 	kthread_stop(con->kthread);  in nbcon_kthread_stop()
1288 	con->kthread = NULL;  in nbcon_kthread_stop()
1293  * @con:	Console to operate on
1296  *		Otherwise false and @con must not be registered.
1303  * If @con was already registered and this function fails, @con must be
1307 bool nbcon_kthread_create(struct console *con)  in nbcon_kthread_create()  argument
1313 	if (con->kthread)  in nbcon_kthread_create()
1316 	kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index);  in nbcon_kthread_create()
1318 		con_printk(KERN_ERR, con, "failed to start printing thread\n");  in nbcon_kthread_create()
1322 	con->kthread = kt;  in nbcon_kthread_create()
1328 	sched_set_normal(con->kthread, -20);  in nbcon_kthread_create()
1391  * @con:	The console to print on
1418 bool nbcon_legacy_emit_next_record(struct console *con, bool *handover,  in nbcon_legacy_emit_next_record()  argument
1426 	ctxt->console	= con;  in nbcon_legacy_emit_next_record()
1456  * @con:			The nbcon console to flush
1460  * Return:	0 if @con was flushed up to @stop_seq Otherwise, error code on
1477 static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq,  in __nbcon_atomic_flush_pending_con()  argument
1484 	ctxt->console			= con;  in __nbcon_atomic_flush_pending_con()
1492 	while (nbcon_seq_read(con) < stop_seq) {  in __nbcon_atomic_flush_pending_con()
1503 			if (nbcon_seq_read(con) < stop_seq)  in __nbcon_atomic_flush_pending_con()
1516  * @con:			The nbcon console to flush
1525 static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq,  in nbcon_atomic_flush_pending_con()  argument
1541 	err = __nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover);  in nbcon_atomic_flush_pending_con()
1563 	    prb_read_valid(prb, nbcon_seq_read(con), NULL)) {  in nbcon_atomic_flush_pending_con()
1577 	struct console *con;  in __nbcon_atomic_flush_pending()  local
1581 	for_each_console_srcu(con) {  in __nbcon_atomic_flush_pending()
1582 		short flags = console_srcu_read_flags(con);  in __nbcon_atomic_flush_pending()
1587 		if (!console_is_usable(con, flags, true))  in __nbcon_atomic_flush_pending()
1590 		if (nbcon_seq_read(con) >= stop_seq)  in __nbcon_atomic_flush_pending()
1593 		nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover);  in __nbcon_atomic_flush_pending()
1662  * @con:	Console to initialize
1665  *		Otherwise @con must not be registered.
1670 bool nbcon_alloc(struct console *con)  in nbcon_alloc()  argument
1675 	if (WARN_ON(!con->write_thread))  in nbcon_alloc()
1678 	rcuwait_init(&con->rcuwait);  in nbcon_alloc()
1679 	init_irq_work(&con->irq_work, nbcon_irq_work);  in nbcon_alloc()
1680 	atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL);  in nbcon_alloc()
1681 	nbcon_state_set(con, &state);  in nbcon_alloc()
1688 	atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), ULSEQ_MAX(prb));  in nbcon_alloc()
1690 	if (con->flags & CON_BOOT) {  in nbcon_alloc()
1696 		con->pbufs = &printk_shared_pbufs;  in nbcon_alloc()
1698 		con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL);  in nbcon_alloc()
1699 		if (!con->pbufs) {  in nbcon_alloc()
1700 			con_printk(KERN_ERR, con, "failed to allocate printing buffer\n");  in nbcon_alloc()
1705 			if (!nbcon_kthread_create(con)) {  in nbcon_alloc()
1706 				kfree(con->pbufs);  in nbcon_alloc()
1707 				con->pbufs = NULL;  in nbcon_alloc()
1718  * @con:	Console to free/cleanup nbcon data
1720 void nbcon_free(struct console *con)  in nbcon_free()  argument
1725 		nbcon_kthread_stop(con);  in nbcon_free()
1727 	nbcon_state_set(con, &state);  in nbcon_free()
1730 	if (!(con->flags & CON_BOOT))  in nbcon_free()
1731 		kfree(con->pbufs);  in nbcon_free()
1733 	con->pbufs = NULL;  in nbcon_free()
1739  * @con:	The nbcon console to acquire
1742  *		@con->device_lock() including disabling migration.
1755 bool nbcon_device_try_acquire(struct console *con)  in nbcon_device_try_acquire()  argument
1757 	struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);  in nbcon_device_try_acquire()
1762 	ctxt->console	= con;  in nbcon_device_try_acquire()
1777  * @con:	The nbcon console acquired in nbcon_device_try_acquire()
1779 void nbcon_device_release(struct console *con)  in nbcon_device_release()  argument
1781 	struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);  in nbcon_device_release()
1798 	if (console_is_usable(con, console_srcu_read_flags(con), true) &&  in nbcon_device_release()
1800 	    prb_read_valid(prb, nbcon_seq_read(con), NULL)) {  in nbcon_device_release()
1806 			__nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb), false);  in nbcon_device_release()