1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _KERNEL_PRINTK_RINGBUFFER_H 4 #define _KERNEL_PRINTK_RINGBUFFER_H 5 6 #include <linux/atomic.h> 7 #include <linux/bits.h> 8 #include <linux/dev_printk.h> 9 #include <linux/stddef.h> 10 #include <linux/types.h> 11 12 /* 13 * Meta information about each stored message. 14 * 15 * All fields are set by the printk code except for @seq, which is 16 * set by the ringbuffer code. 17 */ 18 struct printk_info { 19 u64 seq; /* sequence number */ 20 u64 ts_nsec; /* timestamp in nanoseconds */ 21 u16 text_len; /* length of text message */ 22 u8 facility; /* syslog facility */ 23 u8 flags:5; /* internal record flags */ 24 u8 level:3; /* syslog level */ 25 u32 caller_id; /* thread id or processor id */ 26 27 struct dev_printk_info dev_info; 28 }; 29 30 /* 31 * A structure providing the buffers, used by writers and readers. 32 * 33 * Writers: 34 * Using prb_rec_init_wr(), a writer sets @text_buf_size before calling 35 * prb_reserve(). On success, prb_reserve() sets @info and @text_buf to 36 * buffers reserved for that writer. 37 * 38 * Readers: 39 * Using prb_rec_init_rd(), a reader sets all fields before calling 40 * prb_read_valid(). Note that the reader provides the @info and @text_buf, 41 * buffers. On success, the struct pointed to by @info will be filled and 42 * the char array pointed to by @text_buf will be filled with text data. 43 */ 44 struct printk_record { 45 struct printk_info *info; 46 char *text_buf; 47 unsigned int text_buf_size; 48 }; 49 50 /* Specifies the logical position and span of a data block. */ 51 struct prb_data_blk_lpos { 52 unsigned long begin; 53 unsigned long next; 54 }; 55 56 /* 57 * A descriptor: the complete meta-data for a record. 58 * 59 * @state_var: A bitwise combination of descriptor ID and descriptor state. 60 */ 61 struct prb_desc { 62 atomic_long_t state_var; 63 struct prb_data_blk_lpos text_blk_lpos; 64 }; 65 66 /* A ringbuffer of "ID + data" elements. */ 67 struct prb_data_ring { 68 unsigned int size_bits; 69 char *data; 70 atomic_long_t head_lpos; 71 atomic_long_t tail_lpos; 72 }; 73 74 /* A ringbuffer of "struct prb_desc" elements. */ 75 struct prb_desc_ring { 76 unsigned int count_bits; 77 struct prb_desc *descs; 78 struct printk_info *infos; 79 atomic_long_t head_id; 80 atomic_long_t tail_id; 81 atomic_long_t last_finalized_seq; 82 }; 83 84 /* 85 * The high level structure representing the printk ringbuffer. 86 * 87 * @fail: Count of failed prb_reserve() calls where not even a data-less 88 * record was created. 89 */ 90 struct printk_ringbuffer { 91 struct prb_desc_ring desc_ring; 92 struct prb_data_ring text_data_ring; 93 atomic_long_t fail; 94 }; 95 96 /* 97 * Used by writers as a reserve/commit handle. 98 * 99 * @rb: Ringbuffer where the entry is reserved. 100 * @irqflags: Saved irq flags to restore on entry commit. 101 * @id: ID of the reserved descriptor. 102 * @text_space: Total occupied buffer space in the text data ring, including 103 * ID, alignment padding, and wrapping data blocks. 104 * 105 * This structure is an opaque handle for writers. Its contents are only 106 * to be used by the ringbuffer implementation. 107 */ 108 struct prb_reserved_entry { 109 struct printk_ringbuffer *rb; 110 unsigned long irqflags; 111 unsigned long id; 112 unsigned int text_space; 113 }; 114 115 /* The possible responses of a descriptor state-query. */ 116 enum desc_state { 117 desc_miss = -1, /* ID mismatch (pseudo state) */ 118 desc_reserved = 0x0, /* reserved, in use by writer */ 119 desc_committed = 0x1, /* committed by writer, could get reopened */ 120 desc_finalized = 0x2, /* committed, no further modification allowed */ 121 desc_reusable = 0x3, /* free, not yet used by any writer */ 122 }; 123 124 #define _DATA_SIZE(sz_bits) (1UL << (sz_bits)) 125 #define _DESCS_COUNT(ct_bits) (1U << (ct_bits)) 126 #define DESC_SV_BITS BITS_PER_LONG 127 #define DESC_FLAGS_SHIFT (DESC_SV_BITS - 2) 128 #define DESC_FLAGS_MASK (3UL << DESC_FLAGS_SHIFT) 129 #define DESC_STATE(sv) (3UL & (sv >> DESC_FLAGS_SHIFT)) 130 #define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) | id) 131 #define DESC_ID_MASK (~DESC_FLAGS_MASK) 132 #define DESC_ID(sv) ((sv) & DESC_ID_MASK) 133 134 /* 135 * Special data block logical position values (for fields of 136 * @prb_desc.text_blk_lpos). 137 * 138 * - Bit0 is used to identify if the record has no data block. (Implemented in 139 * the LPOS_DATALESS() macro.) 140 * 141 * - Bit1 specifies the reason for not having a data block. 142 * 143 * These special values could never be real lpos values because of the 144 * meta data and alignment padding of data blocks. (See to_blk_size() for 145 * details.) 146 */ 147 #define FAILED_LPOS 0x1 148 #define EMPTY_LINE_LPOS 0x3 149 150 #define FAILED_BLK_LPOS \ 151 { \ 152 .begin = FAILED_LPOS, \ 153 .next = FAILED_LPOS, \ 154 } 155 156 /* 157 * Descriptor Bootstrap 158 * 159 * The descriptor array is minimally initialized to allow immediate usage 160 * by readers and writers. The requirements that the descriptor array 161 * initialization must satisfy: 162 * 163 * Req1 164 * The tail must point to an existing (committed or reusable) descriptor. 165 * This is required by the implementation of prb_first_seq(). 166 * 167 * Req2 168 * Readers must see that the ringbuffer is initially empty. 169 * 170 * Req3 171 * The first record reserved by a writer is assigned sequence number 0. 172 * 173 * To satisfy Req1, the tail initially points to a descriptor that is 174 * minimally initialized (having no data block, i.e. data-less with the 175 * data block's lpos @begin and @next values set to FAILED_LPOS). 176 * 177 * To satisfy Req2, the initial tail descriptor is initialized to the 178 * reusable state. Readers recognize reusable descriptors as existing 179 * records, but skip over them. 180 * 181 * To satisfy Req3, the last descriptor in the array is used as the initial 182 * head (and tail) descriptor. This allows the first record reserved by a 183 * writer (head + 1) to be the first descriptor in the array. (Only the first 184 * descriptor in the array could have a valid sequence number of 0.) 185 * 186 * The first time a descriptor is reserved, it is assigned a sequence number 187 * with the value of the array index. A "first time reserved" descriptor can 188 * be recognized because it has a sequence number of 0 but does not have an 189 * index of 0. (Only the first descriptor in the array could have a valid 190 * sequence number of 0.) After the first reservation, all future reservations 191 * (recycling) simply involve incrementing the sequence number by the array 192 * count. 193 * 194 * Hack #1 195 * Only the first descriptor in the array is allowed to have the sequence 196 * number 0. In this case it is not possible to recognize if it is being 197 * reserved the first time (set to index value) or has been reserved 198 * previously (increment by the array count). This is handled by _always_ 199 * incrementing the sequence number by the array count when reserving the 200 * first descriptor in the array. In order to satisfy Req3, the sequence 201 * number of the first descriptor in the array is initialized to minus 202 * the array count. Then, upon the first reservation, it is incremented 203 * to 0, thus satisfying Req3. 204 * 205 * Hack #2 206 * prb_first_seq() can be called at any time by readers to retrieve the 207 * sequence number of the tail descriptor. However, due to Req2 and Req3, 208 * initially there are no records to report the sequence number of 209 * (sequence numbers are u64 and there is nothing less than 0). To handle 210 * this, the sequence number of the initial tail descriptor is initialized 211 * to 0. Technically this is incorrect, because there is no record with 212 * sequence number 0 (yet) and the tail descriptor is not the first 213 * descriptor in the array. But it allows prb_read_valid() to correctly 214 * report the existence of a record for _any_ given sequence number at all 215 * times. Bootstrapping is complete when the tail is pushed the first 216 * time, thus finally pointing to the first descriptor reserved by a 217 * writer, which has the assigned sequence number 0. 218 */ 219 220 /* 221 * Initiating Logical Value Overflows 222 * 223 * Both logical position (lpos) and ID values can be mapped to array indexes 224 * but may experience overflows during the lifetime of the system. To ensure 225 * that printk_ringbuffer can handle the overflows for these types, initial 226 * values are chosen that map to the correct initial array indexes, but will 227 * result in overflows soon. 228 * 229 * BLK0_LPOS 230 * The initial @head_lpos and @tail_lpos for data rings. It is at index 231 * 0 and the lpos value is such that it will overflow on the first wrap. 232 * 233 * DESC0_ID 234 * The initial @head_id and @tail_id for the desc ring. It is at the last 235 * index of the descriptor array (see Req3 above) and the ID value is such 236 * that it will overflow on the second wrap. 237 */ 238 #define BLK0_LPOS(sz_bits) (-(_DATA_SIZE(sz_bits))) 239 #define DESC0_ID(ct_bits) DESC_ID(-(_DESCS_COUNT(ct_bits) + 1)) 240 #define DESC0_SV(ct_bits) DESC_SV(DESC0_ID(ct_bits), desc_reusable) 241 242 /* 243 * Define a ringbuffer with an external text data buffer. The same as 244 * DEFINE_PRINTKRB() but requires specifying an external buffer for the 245 * text data. 246 * 247 * Note: The specified external buffer must be of the size: 248 * 2 ^ (descbits + avgtextbits) 249 */ 250 #define _DEFINE_PRINTKRB(name, descbits, avgtextbits, text_buf) \ 251 static struct prb_desc _##name##_descs[_DESCS_COUNT(descbits)] = { \ 252 /* the initial head and tail */ \ 253 [_DESCS_COUNT(descbits) - 1] = { \ 254 /* reusable */ \ 255 .state_var = ATOMIC_INIT(DESC0_SV(descbits)), \ 256 /* no associated data block */ \ 257 .text_blk_lpos = FAILED_BLK_LPOS, \ 258 }, \ 259 }; \ 260 static struct printk_info _##name##_infos[_DESCS_COUNT(descbits)] = { \ 261 /* this will be the first record reserved by a writer */ \ 262 [0] = { \ 263 /* will be incremented to 0 on the first reservation */ \ 264 .seq = -(u64)_DESCS_COUNT(descbits), \ 265 }, \ 266 /* the initial head and tail */ \ 267 [_DESCS_COUNT(descbits) - 1] = { \ 268 /* reports the first seq value during the bootstrap phase */ \ 269 .seq = 0, \ 270 }, \ 271 }; \ 272 static struct printk_ringbuffer name = { \ 273 .desc_ring = { \ 274 .count_bits = descbits, \ 275 .descs = &_##name##_descs[0], \ 276 .infos = &_##name##_infos[0], \ 277 .head_id = ATOMIC_INIT(DESC0_ID(descbits)), \ 278 .tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \ 279 .last_finalized_seq = ATOMIC_INIT(0), \ 280 }, \ 281 .text_data_ring = { \ 282 .size_bits = (avgtextbits) + (descbits), \ 283 .data = text_buf, \ 284 .head_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \ 285 .tail_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \ 286 }, \ 287 .fail = ATOMIC_LONG_INIT(0), \ 288 } 289 290 /** 291 * DEFINE_PRINTKRB() - Define a ringbuffer. 292 * 293 * @name: The name of the ringbuffer variable. 294 * @descbits: The number of descriptors as a power-of-2 value. 295 * @avgtextbits: The average text data size per record as a power-of-2 value. 296 * 297 * This is a macro for defining a ringbuffer and all internal structures 298 * such that it is ready for immediate use. See _DEFINE_PRINTKRB() for a 299 * variant where the text data buffer can be specified externally. 300 */ 301 #define DEFINE_PRINTKRB(name, descbits, avgtextbits) \ 302 static char _##name##_text[1U << ((avgtextbits) + (descbits))] \ 303 __aligned(__alignof__(unsigned long)); \ 304 _DEFINE_PRINTKRB(name, descbits, avgtextbits, &_##name##_text[0]) 305 306 /* Writer Interface */ 307 308 /** 309 * prb_rec_init_wr() - Initialize a buffer for writing records. 310 * 311 * @r: The record to initialize. 312 * @text_buf_size: The needed text buffer size. 313 */ prb_rec_init_wr(struct printk_record * r,unsigned int text_buf_size)314 static inline void prb_rec_init_wr(struct printk_record *r, 315 unsigned int text_buf_size) 316 { 317 r->info = NULL; 318 r->text_buf = NULL; 319 r->text_buf_size = text_buf_size; 320 } 321 322 bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb, 323 struct printk_record *r); 324 bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb, 325 struct printk_record *r, u32 caller_id, unsigned int max_size); 326 void prb_commit(struct prb_reserved_entry *e); 327 void prb_final_commit(struct prb_reserved_entry *e); 328 329 void prb_init(struct printk_ringbuffer *rb, 330 char *text_buf, unsigned int text_buf_size, 331 struct prb_desc *descs, unsigned int descs_count_bits, 332 struct printk_info *infos); 333 unsigned int prb_record_text_space(struct prb_reserved_entry *e); 334 335 /* Reader Interface */ 336 337 /** 338 * prb_rec_init_rd() - Initialize a buffer for reading records. 339 * 340 * @r: The record to initialize. 341 * @info: A buffer to store record meta-data. 342 * @text_buf: A buffer to store text data. 343 * @text_buf_size: The size of @text_buf. 344 * 345 * Initialize all the fields that a reader is interested in. All arguments 346 * (except @r) are optional. Only record data for arguments that are 347 * non-NULL or non-zero will be read. 348 */ prb_rec_init_rd(struct printk_record * r,struct printk_info * info,char * text_buf,unsigned int text_buf_size)349 static inline void prb_rec_init_rd(struct printk_record *r, 350 struct printk_info *info, 351 char *text_buf, unsigned int text_buf_size) 352 { 353 r->info = info; 354 r->text_buf = text_buf; 355 r->text_buf_size = text_buf_size; 356 } 357 358 /** 359 * prb_for_each_record() - Iterate over the records of a ringbuffer. 360 * 361 * @from: The sequence number to begin with. 362 * @rb: The ringbuffer to iterate over. 363 * @s: A u64 to store the sequence number on each iteration. 364 * @r: A printk_record to store the record on each iteration. 365 * 366 * This is a macro for conveniently iterating over a ringbuffer. 367 * Note that @s may not be the sequence number of the record on each 368 * iteration. For the sequence number, @r->info->seq should be checked. 369 * 370 * Context: Any context. 371 */ 372 #define prb_for_each_record(from, rb, s, r) \ 373 for ((s) = from; prb_read_valid(rb, s, r); (s) = (r)->info->seq + 1) 374 375 /** 376 * prb_for_each_info() - Iterate over the meta data of a ringbuffer. 377 * 378 * @from: The sequence number to begin with. 379 * @rb: The ringbuffer to iterate over. 380 * @s: A u64 to store the sequence number on each iteration. 381 * @i: A printk_info to store the record meta data on each iteration. 382 * @lc: An unsigned int to store the text line count of each record. 383 * 384 * This is a macro for conveniently iterating over a ringbuffer. 385 * Note that @s may not be the sequence number of the record on each 386 * iteration. For the sequence number, @r->info->seq should be checked. 387 * 388 * Context: Any context. 389 */ 390 #define prb_for_each_info(from, rb, s, i, lc) \ 391 for ((s) = from; prb_read_valid_info(rb, s, i, lc); (s) = (i)->seq + 1) 392 393 bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, 394 struct printk_record *r); 395 bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, 396 struct printk_info *info, unsigned int *line_count); 397 398 u64 prb_first_seq(struct printk_ringbuffer *rb); 399 u64 prb_first_valid_seq(struct printk_ringbuffer *rb); 400 u64 prb_next_seq(struct printk_ringbuffer *rb); 401 u64 prb_next_reserve_seq(struct printk_ringbuffer *rb); 402 403 #ifdef CONFIG_64BIT 404 405 #define __u64seq_to_ulseq(u64seq) (u64seq) 406 #define __ulseq_to_u64seq(rb, ulseq) (ulseq) 407 #define ULSEQ_MAX(rb) (-1) 408 409 #else /* CONFIG_64BIT */ 410 411 #define __u64seq_to_ulseq(u64seq) ((u32)u64seq) 412 #define ULSEQ_MAX(rb) __u64seq_to_ulseq(prb_first_seq(rb) + 0x80000000UL) 413 __ulseq_to_u64seq(struct printk_ringbuffer * rb,u32 ulseq)414 static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) 415 { 416 u64 rb_first_seq = prb_first_seq(rb); 417 u64 seq; 418 419 /* 420 * The provided sequence is only the lower 32 bits of the ringbuffer 421 * sequence. It needs to be expanded to 64bit. Get the first sequence 422 * number from the ringbuffer and fold it. 423 * 424 * Having a 32bit representation in the console is sufficient. 425 * If a console ever gets more than 2^31 records behind 426 * the ringbuffer then this is the least of the problems. 427 * 428 * Also the access to the ring buffer is always safe. 429 */ 430 seq = rb_first_seq - (s32)((u32)rb_first_seq - ulseq); 431 432 return seq; 433 } 434 435 #endif /* CONFIG_64BIT */ 436 437 #endif /* _KERNEL_PRINTK_RINGBUFFER_H */ 438