315 #define DESCS_COUNT(desc_ring)		_DESCS_COUNT((desc_ring)->count_bits)  argument
316 #define DESCS_COUNT_MASK(desc_ring)	(DESCS_COUNT(desc_ring) - 1)  argument
322 #define DESC_INDEX(desc_ring, n)	((n) & DESCS_COUNT_MASK(desc_ring))  argument
337 #define DESC_ID_PREV_WRAP(desc_ring, id) \  argument
338 DESC_ID((id) - DESCS_COUNT(desc_ring))
359 static struct prb_desc *to_desc(struct prb_desc_ring *desc_ring, u64 n)  in to_desc()  argument
361 	return &desc_ring->descs[DESC_INDEX(desc_ring, n)];  in to_desc()
368 static struct printk_info *to_info(struct prb_desc_ring *desc_ring, u64 n)  in to_info()  argument
370 	return &desc_ring->infos[DESC_INDEX(desc_ring, n)];  in to_info()
436 static enum desc_state desc_read(struct prb_desc_ring *desc_ring,  in desc_read()  argument
440 	struct printk_info *info = to_info(desc_ring, id);  in desc_read()
441 	struct prb_desc *desc = to_desc(desc_ring, id);  in desc_read()
547 static void desc_make_reusable(struct prb_desc_ring *desc_ring,  in desc_make_reusable()  argument
552 	struct prb_desc *desc = to_desc(desc_ring, id);  in desc_make_reusable()
575 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in data_make_reusable()  local
596 		d_state = desc_read(desc_ring, id, &desc,  in data_make_reusable()
611 			desc_make_reusable(desc_ring, id);  in data_make_reusable()
774 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in desc_push_tail()  local
778 	d_state = desc_read(desc_ring, tail_id, &desc, NULL, NULL);  in desc_push_tail()
788 		    DESC_ID_PREV_WRAP(desc_ring, tail_id)) {  in desc_push_tail()
803 		desc_make_reusable(desc_ring, tail_id);  in desc_push_tail()
828 	d_state = desc_read(desc_ring, DESC_ID(tail_id + 1), &desc,  in desc_push_tail()
839 		atomic_long_cmpxchg(&desc_ring->tail_id, tail_id,  in desc_push_tail()
871 		if (atomic_long_read(&desc_ring->tail_id) == tail_id) /* LMM(desc_push_tail:D) */  in desc_push_tail()
881 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in desc_reserve()  local
888 	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(desc_reserve:A) */  in desc_reserve()
892 		id_prev_wrap = DESC_ID_PREV_WRAP(desc_ring, id);  in desc_reserve()
918 		if (id_prev_wrap == atomic_long_read(&desc_ring->tail_id  in desc_reserve()
969 	} while (!atomic_long_try_cmpxchg(&desc_ring->head_id, &head_id,  in desc_reserve()
972 	desc = to_desc(desc_ring, id);  in desc_reserve()
1275 static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring,  in desc_reopen_last()  argument
1285 	id = atomic_long_read(&desc_ring->head_id);  in desc_reopen_last()
1291 	d_state = desc_read(desc_ring, id, &desc, NULL, &cid);  in desc_reopen_last()
1295 	d = to_desc(desc_ring, id);  in desc_reopen_last()
1369 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_reserve_in_last()  local
1378 	d = desc_reopen_last(desc_ring, caller_id, &id);  in prb_reserve_in_last()
1386 	info = to_info(desc_ring, id);  in prb_reserve_in_last()
1479 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in desc_last_finalized_seq()  local
1487 	ulseq = atomic_long_read_acquire(&desc_ring->last_finalized_seq  in desc_last_finalized_seq()
1503 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in desc_update_last_finalized()  local
1551 	if (!atomic_long_try_cmpxchg_release(&desc_ring->last_finalized_seq,  in desc_update_last_finalized()
1564 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in desc_make_final()  local
1566 	struct prb_desc *d = to_desc(desc_ring, id);  in desc_make_final()
1602 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_reserve()  local
1626 	d = to_desc(desc_ring, id);  in prb_reserve()
1627 	info = to_info(desc_ring, id);  in prb_reserve()
1655 	if (seq == 0 && DESC_INDEX(desc_ring, id) != 0)  in prb_reserve()
1656 		info->seq = DESC_INDEX(desc_ring, id);  in prb_reserve()
1658 		info->seq = seq + DESCS_COUNT(desc_ring);  in prb_reserve()
1692 	struct prb_desc_ring *desc_ring = &e->rb->desc_ring;  in _prb_commit()  local
1693 	struct prb_desc *d = to_desc(desc_ring, e->id);  in _prb_commit()
1748 	struct prb_desc_ring *desc_ring = &e->rb->desc_ring;  in prb_commit()  local
1758 	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */  in prb_commit()
1866 static int desc_read_finalized_seq(struct prb_desc_ring *desc_ring,  in desc_read_finalized_seq()  argument
1874 	d_state = desc_read(desc_ring, id, desc_out, &s, NULL);  in desc_read_finalized_seq()
1910 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_read()  local
1911 	struct printk_info *info = to_info(desc_ring, seq);  in prb_read()
1912 	struct prb_desc *rdesc = to_desc(desc_ring, seq);  in prb_read()
1922 	err = desc_read_finalized_seq(desc_ring, id, seq, &desc);  in prb_read()
1942 	return desc_read_finalized_seq(desc_ring, id, seq, &desc);  in prb_read()
1948 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_first_seq()  local
1955 		id = atomic_long_read(&rb->desc_ring.tail_id); /* LMM(prb_first_seq:A) */  in prb_first_seq()
1957 		d_state = desc_read(desc_ring, id, &desc, &seq, NULL); /* LMM(prb_first_seq:B) */  in prb_first_seq()
2007 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_next_reserve_seq()  local
2047 	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_next_reserve_seq:A) */  in prb_next_reserve_seq()
2049 	d = to_desc(desc_ring, last_finalized_seq);  in prb_next_reserve_seq()
2056 	err = desc_read_finalized_seq(desc_ring, last_finalized_id, last_finalized_seq, &desc);  in prb_next_reserve_seq()
2068 			if (head_id == DESC0_ID(desc_ring->count_bits))  in prb_next_reserve_seq()
2076 			last_finalized_id = DESC0_ID(desc_ring->count_bits) + 1;  in prb_next_reserve_seq()
2312 	rb->desc_ring.count_bits = descbits;  in prb_init()
2313 	rb->desc_ring.descs = descs;  in prb_init()
2314 	rb->desc_ring.infos = infos;  in prb_init()
2315 	atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits));  in prb_init()
2316 	atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits));  in prb_init()
2317 	atomic_long_set(&rb->desc_ring.last_finalized_seq, 0);  in prb_init()