1 /* 2 * Copyright (c) 2014-2021 The Linux Foundation. All rights reserved. 3 * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for 6 * any purpose with or without fee is hereby granted, provided that the 7 * above copyright notice and this permission notice appear in all 8 * copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 17 * PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 /** 21 * DOC: qdf_mem 22 * This file provides OS dependent memory management APIs 23 */ 24 25 #include "qdf_debugfs.h" 26 #include "qdf_mem.h" 27 #include "qdf_nbuf.h" 28 #include "qdf_lock.h" 29 #include "qdf_mc_timer.h" 30 #include "qdf_module.h" 31 #include <qdf_trace.h> 32 #include "qdf_str.h" 33 #include "qdf_talloc.h" 34 #include <linux/debugfs.h> 35 #include <linux/seq_file.h> 36 #include <linux/string.h> 37 #include <qdf_list.h> 38 39 #ifdef CNSS_MEM_PRE_ALLOC 40 #ifdef CONFIG_CNSS_OUT_OF_TREE 41 #include "cnss_prealloc.h" 42 #else 43 #include <net/cnss_prealloc.h> 44 #endif 45 #endif 46 47 #if defined(MEMORY_DEBUG) || defined(NBUF_MEMORY_DEBUG) 48 static bool mem_debug_disabled; 49 qdf_declare_param(mem_debug_disabled, bool); 50 qdf_export_symbol(mem_debug_disabled); 51 #endif 52 53 #ifdef MEMORY_DEBUG 54 static bool is_initial_mem_debug_disabled; 55 #endif 56 57 /* Preprocessor Definitions and Constants */ 58 #define QDF_MEM_MAX_MALLOC (4096 * 1024) /* 4 Mega Bytes */ 59 #define QDF_MEM_WARN_THRESHOLD 300 /* ms */ 60 #define QDF_DEBUG_STRING_SIZE 512 61 62 /** 63 * struct __qdf_mem_stat - qdf memory statistics 64 * @kmalloc: total kmalloc allocations 65 * @dma: total dma allocations 66 * @skb: total skb allocations 67 * @skb_total: total skb allocations in host driver 68 * @dp_tx_skb: total Tx skb allocations in datapath 69 * @dp_rx_skb: total Rx skb allocations in datapath 70 * @skb_mem_max: high watermark for skb allocations 71 * @dp_tx_skb_mem_max: high watermark for Tx DP skb allocations 72 * @dp_rx_skb_mem_max: high watermark for Rx DP skb allocations 73 * @dp_tx_skb_count: DP Tx buffer count 74 * @dp_tx_skb_count_max: High watermark for DP Tx buffer count 75 * @dp_rx_skb_count: DP Rx buffer count 76 * @dp_rx_skb_count_max: High watermark for DP Rx buffer count 77 * @tx_descs_outstanding: Current pending Tx descs count 78 * @tx_descs_max: High watermark for pending Tx descs count 79 */ 80 static struct __qdf_mem_stat { 81 qdf_atomic_t kmalloc; 82 qdf_atomic_t dma; 83 qdf_atomic_t skb; 84 qdf_atomic_t skb_total; 85 qdf_atomic_t dp_tx_skb; 86 qdf_atomic_t dp_rx_skb; 87 int32_t skb_mem_max; 88 int32_t dp_tx_skb_mem_max; 89 int32_t dp_rx_skb_mem_max; 90 qdf_atomic_t dp_tx_skb_count; 91 int32_t dp_tx_skb_count_max; 92 qdf_atomic_t dp_rx_skb_count; 93 int32_t dp_rx_skb_count_max; 94 qdf_atomic_t tx_descs_outstanding; 95 int32_t tx_descs_max; 96 } qdf_mem_stat; 97 98 #ifdef MEMORY_DEBUG 99 #include "qdf_debug_domain.h" 100 101 enum list_type { 102 LIST_TYPE_MEM = 0, 103 LIST_TYPE_DMA = 1, 104 LIST_TYPE_NBUF = 2, 105 LIST_TYPE_MAX, 106 }; 107 108 /** 109 * major_alloc_priv: private data registered to debugfs entry created to list 110 * the list major allocations 111 * @type: type of the list to be parsed 112 * @threshold: configured by user by overwriting the respective debugfs 113 * sys entry. This is to list the functions which requested 114 * memory/dma allocations more than threshold nubmer of times. 115 */ 116 struct major_alloc_priv { 117 enum list_type type; 118 uint32_t threshold; 119 }; 120 121 static qdf_list_t qdf_mem_domains[QDF_DEBUG_DOMAIN_COUNT]; 122 static qdf_spinlock_t qdf_mem_list_lock; 123 124 static qdf_list_t qdf_mem_dma_domains[QDF_DEBUG_DOMAIN_COUNT]; 125 static qdf_spinlock_t qdf_mem_dma_list_lock; 126 127 static inline qdf_list_t *qdf_mem_list_get(enum qdf_debug_domain domain) 128 { 129 return &qdf_mem_domains[domain]; 130 } 131 132 static inline qdf_list_t *qdf_mem_dma_list(enum qdf_debug_domain domain) 133 { 134 return &qdf_mem_dma_domains[domain]; 135 } 136 137 /** 138 * struct qdf_mem_header - memory object to dubug 139 * @node: node to the list 140 * @domain: the active memory domain at time of allocation 141 * @freed: flag set during free, used to detect double frees 142 * Use uint8_t so we can detect corruption 143 * @func: name of the function the allocation was made from 144 * @line: line number of the file the allocation was made from 145 * @size: size of the allocation in bytes 146 * @caller: Caller of the function for which memory is allocated 147 * @header: a known value, used to detect out-of-bounds access 148 * @time: timestamp at which allocation was made 149 */ 150 struct qdf_mem_header { 151 qdf_list_node_t node; 152 enum qdf_debug_domain domain; 153 uint8_t freed; 154 char func[QDF_MEM_FUNC_NAME_SIZE]; 155 uint32_t line; 156 uint32_t size; 157 void *caller; 158 uint64_t header; 159 uint64_t time; 160 }; 161 162 /* align the qdf_mem_header to 8 bytes */ 163 #define QDF_DMA_MEM_HEADER_ALIGN 8 164 165 static uint64_t WLAN_MEM_HEADER = 0x6162636465666768; 166 static uint64_t WLAN_MEM_TRAILER = 0x8081828384858687; 167 168 static inline struct qdf_mem_header *qdf_mem_get_header(void *ptr) 169 { 170 return (struct qdf_mem_header *)ptr - 1; 171 } 172 173 /* make sure the header pointer is 8bytes aligned */ 174 static inline struct qdf_mem_header *qdf_mem_dma_get_header(void *ptr, 175 qdf_size_t size) 176 { 177 return (struct qdf_mem_header *) 178 qdf_roundup((size_t)((uint8_t *)ptr + size), 179 QDF_DMA_MEM_HEADER_ALIGN); 180 } 181 182 static inline uint64_t *qdf_mem_get_trailer(struct qdf_mem_header *header) 183 { 184 return (uint64_t *)((void *)(header + 1) + header->size); 185 } 186 187 static inline void *qdf_mem_get_ptr(struct qdf_mem_header *header) 188 { 189 return (void *)(header + 1); 190 } 191 192 /* number of bytes needed for the qdf memory debug information */ 193 #define QDF_MEM_DEBUG_SIZE \ 194 (sizeof(struct qdf_mem_header) + sizeof(WLAN_MEM_TRAILER)) 195 196 /* number of bytes needed for the qdf dma memory debug information */ 197 #define QDF_DMA_MEM_DEBUG_SIZE \ 198 (sizeof(struct qdf_mem_header) + QDF_DMA_MEM_HEADER_ALIGN) 199 200 static void qdf_mem_trailer_init(struct qdf_mem_header *header) 201 { 202 QDF_BUG(header); 203 if (!header) 204 return; 205 *qdf_mem_get_trailer(header) = WLAN_MEM_TRAILER; 206 } 207 208 static void qdf_mem_header_init(struct qdf_mem_header *header, qdf_size_t size, 209 const char *func, uint32_t line, void *caller) 210 { 211 QDF_BUG(header); 212 if (!header) 213 return; 214 215 header->domain = qdf_debug_domain_get(); 216 header->freed = false; 217 218 qdf_str_lcopy(header->func, func, QDF_MEM_FUNC_NAME_SIZE); 219 220 header->line = line; 221 header->size = size; 222 header->caller = caller; 223 header->header = WLAN_MEM_HEADER; 224 header->time = qdf_get_log_timestamp(); 225 } 226 227 enum qdf_mem_validation_bitmap { 228 QDF_MEM_BAD_HEADER = 1 << 0, 229 QDF_MEM_BAD_TRAILER = 1 << 1, 230 QDF_MEM_BAD_SIZE = 1 << 2, 231 QDF_MEM_DOUBLE_FREE = 1 << 3, 232 QDF_MEM_BAD_FREED = 1 << 4, 233 QDF_MEM_BAD_NODE = 1 << 5, 234 QDF_MEM_BAD_DOMAIN = 1 << 6, 235 QDF_MEM_WRONG_DOMAIN = 1 << 7, 236 }; 237 238 static enum qdf_mem_validation_bitmap 239 qdf_mem_trailer_validate(struct qdf_mem_header *header) 240 { 241 enum qdf_mem_validation_bitmap error_bitmap = 0; 242 243 if (*qdf_mem_get_trailer(header) != WLAN_MEM_TRAILER) 244 error_bitmap |= QDF_MEM_BAD_TRAILER; 245 return error_bitmap; 246 } 247 248 static enum qdf_mem_validation_bitmap 249 qdf_mem_header_validate(struct qdf_mem_header *header, 250 enum qdf_debug_domain domain) 251 { 252 enum qdf_mem_validation_bitmap error_bitmap = 0; 253 254 if (header->header != WLAN_MEM_HEADER) 255 error_bitmap |= QDF_MEM_BAD_HEADER; 256 257 if (header->size > QDF_MEM_MAX_MALLOC) 258 error_bitmap |= QDF_MEM_BAD_SIZE; 259 260 if (header->freed == true) 261 error_bitmap |= QDF_MEM_DOUBLE_FREE; 262 else if (header->freed) 263 error_bitmap |= QDF_MEM_BAD_FREED; 264 265 if (!qdf_list_node_in_any_list(&header->node)) 266 error_bitmap |= QDF_MEM_BAD_NODE; 267 268 if (header->domain < QDF_DEBUG_DOMAIN_INIT || 269 header->domain >= QDF_DEBUG_DOMAIN_COUNT) 270 error_bitmap |= QDF_MEM_BAD_DOMAIN; 271 else if (header->domain != domain) 272 error_bitmap |= QDF_MEM_WRONG_DOMAIN; 273 274 return error_bitmap; 275 } 276 277 static void 278 qdf_mem_header_assert_valid(struct qdf_mem_header *header, 279 enum qdf_debug_domain current_domain, 280 enum qdf_mem_validation_bitmap error_bitmap, 281 const char *func, 282 uint32_t line) 283 { 284 if (!error_bitmap) 285 return; 286 287 if (error_bitmap & QDF_MEM_BAD_HEADER) 288 qdf_err("Corrupted memory header 0x%llx (expected 0x%llx)", 289 header->header, WLAN_MEM_HEADER); 290 291 if (error_bitmap & QDF_MEM_BAD_SIZE) 292 qdf_err("Corrupted memory size %u (expected < %d)", 293 header->size, QDF_MEM_MAX_MALLOC); 294 295 if (error_bitmap & QDF_MEM_BAD_TRAILER) 296 qdf_err("Corrupted memory trailer 0x%llx (expected 0x%llx)", 297 *qdf_mem_get_trailer(header), WLAN_MEM_TRAILER); 298 299 if (error_bitmap & QDF_MEM_DOUBLE_FREE) 300 qdf_err("Memory has previously been freed"); 301 302 if (error_bitmap & QDF_MEM_BAD_FREED) 303 qdf_err("Corrupted memory freed flag 0x%x", header->freed); 304 305 if (error_bitmap & QDF_MEM_BAD_NODE) 306 qdf_err("Corrupted memory header node or double free"); 307 308 if (error_bitmap & QDF_MEM_BAD_DOMAIN) 309 qdf_err("Corrupted memory domain 0x%x", header->domain); 310 311 if (error_bitmap & QDF_MEM_WRONG_DOMAIN) 312 qdf_err("Memory domain mismatch; allocated:%s(%d), current:%s(%d)", 313 qdf_debug_domain_name(header->domain), header->domain, 314 qdf_debug_domain_name(current_domain), current_domain); 315 316 QDF_MEMDEBUG_PANIC("Fatal memory error detected @ %s:%d", func, line); 317 } 318 319 /** 320 * struct __qdf_mem_info - memory statistics 321 * @func: the function which allocated memory 322 * @line: the line at which allocation happened 323 * @size: the size of allocation 324 * @caller: Address of the caller function 325 * @count: how many allocations of same type 326 * @time: timestamp at which allocation happened 327 */ 328 struct __qdf_mem_info { 329 char func[QDF_MEM_FUNC_NAME_SIZE]; 330 uint32_t line; 331 uint32_t size; 332 void *caller; 333 uint32_t count; 334 uint64_t time; 335 }; 336 337 /* 338 * The table depth defines the de-duplication proximity scope. 339 * A deeper table takes more time, so choose any optimum value. 340 */ 341 #define QDF_MEM_STAT_TABLE_SIZE 8 342 343 /** 344 * qdf_mem_debug_print_header() - memory debug header print logic 345 * @print: the print adapter function 346 * @print_priv: the private data to be consumed by @print 347 * @threshold: the threshold value set by user to list top allocations 348 * 349 * Return: None 350 */ 351 static void qdf_mem_debug_print_header(qdf_abstract_print print, 352 void *print_priv, 353 uint32_t threshold) 354 { 355 if (threshold) 356 print(print_priv, "APIs requested allocations >= %u no of time", 357 threshold); 358 print(print_priv, 359 "--------------------------------------------------------------"); 360 print(print_priv, 361 " count size total filename caller timestamp"); 362 print(print_priv, 363 "--------------------------------------------------------------"); 364 } 365 366 /** 367 * qdf_mem_meta_table_insert() - insert memory metadata into the given table 368 * @table: the memory metadata table to insert into 369 * @meta: the memory metadata to insert 370 * 371 * Return: true if the table is full after inserting, false otherwise 372 */ 373 static bool qdf_mem_meta_table_insert(struct __qdf_mem_info *table, 374 struct qdf_mem_header *meta) 375 { 376 int i; 377 378 for (i = 0; i < QDF_MEM_STAT_TABLE_SIZE; i++) { 379 if (!table[i].count) { 380 qdf_str_lcopy(table[i].func, meta->func, 381 QDF_MEM_FUNC_NAME_SIZE); 382 table[i].line = meta->line; 383 table[i].size = meta->size; 384 table[i].count = 1; 385 table[i].caller = meta->caller; 386 table[i].time = meta->time; 387 break; 388 } 389 390 if (qdf_str_eq(table[i].func, meta->func) && 391 table[i].line == meta->line && 392 table[i].size == meta->size && 393 table[i].caller == meta->caller) { 394 table[i].count++; 395 break; 396 } 397 } 398 399 /* return true if the table is now full */ 400 return i >= QDF_MEM_STAT_TABLE_SIZE - 1; 401 } 402 403 /** 404 * qdf_mem_domain_print() - output agnostic memory domain print logic 405 * @domain: the memory domain to print 406 * @print: the print adapter function 407 * @print_priv: the private data to be consumed by @print 408 * @threshold: the threshold value set by uset to list top allocations 409 * @mem_print: pointer to function which prints the memory allocation data 410 * 411 * Return: None 412 */ 413 static void qdf_mem_domain_print(qdf_list_t *domain, 414 qdf_abstract_print print, 415 void *print_priv, 416 uint32_t threshold, 417 void (*mem_print)(struct __qdf_mem_info *, 418 qdf_abstract_print, 419 void *, uint32_t)) 420 { 421 QDF_STATUS status; 422 struct __qdf_mem_info table[QDF_MEM_STAT_TABLE_SIZE]; 423 qdf_list_node_t *node; 424 425 qdf_mem_zero(table, sizeof(table)); 426 qdf_mem_debug_print_header(print, print_priv, threshold); 427 428 /* hold lock while inserting to avoid use-after free of the metadata */ 429 qdf_spin_lock(&qdf_mem_list_lock); 430 status = qdf_list_peek_front(domain, &node); 431 while (QDF_IS_STATUS_SUCCESS(status)) { 432 struct qdf_mem_header *meta = (struct qdf_mem_header *)node; 433 bool is_full = qdf_mem_meta_table_insert(table, meta); 434 435 qdf_spin_unlock(&qdf_mem_list_lock); 436 437 if (is_full) { 438 (*mem_print)(table, print, print_priv, threshold); 439 qdf_mem_zero(table, sizeof(table)); 440 } 441 442 qdf_spin_lock(&qdf_mem_list_lock); 443 status = qdf_list_peek_next(domain, node, &node); 444 } 445 qdf_spin_unlock(&qdf_mem_list_lock); 446 447 (*mem_print)(table, print, print_priv, threshold); 448 } 449 450 /** 451 * qdf_mem_meta_table_print() - memory metadata table print logic 452 * @table: the memory metadata table to print 453 * @print: the print adapter function 454 * @print_priv: the private data to be consumed by @print 455 * @threshold: the threshold value set by user to list top allocations 456 * 457 * Return: None 458 */ 459 static void qdf_mem_meta_table_print(struct __qdf_mem_info *table, 460 qdf_abstract_print print, 461 void *print_priv, 462 uint32_t threshold) 463 { 464 int i; 465 char debug_str[QDF_DEBUG_STRING_SIZE]; 466 size_t len = 0; 467 char *debug_prefix = "WLAN_BUG_RCA: memory leak detected"; 468 469 len += qdf_scnprintf(debug_str, sizeof(debug_str) - len, 470 "%s", debug_prefix); 471 472 for (i = 0; i < QDF_MEM_STAT_TABLE_SIZE; i++) { 473 if (!table[i].count) 474 break; 475 476 print(print_priv, 477 "%6u x %5u = %7uB @ %s:%u %pS %llu", 478 table[i].count, 479 table[i].size, 480 table[i].count * table[i].size, 481 table[i].func, 482 table[i].line, table[i].caller, 483 table[i].time); 484 len += qdf_scnprintf(debug_str + len, 485 sizeof(debug_str) - len, 486 " @ %s:%u %pS", 487 table[i].func, 488 table[i].line, 489 table[i].caller); 490 } 491 print(print_priv, "%s", debug_str); 492 } 493 494 static int qdf_err_printer(void *priv, const char *fmt, ...) 495 { 496 va_list args; 497 498 va_start(args, fmt); 499 QDF_VTRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, (char *)fmt, args); 500 va_end(args); 501 502 return 0; 503 } 504 505 #endif /* MEMORY_DEBUG */ 506 507 bool prealloc_disabled = 1; 508 qdf_declare_param(prealloc_disabled, bool); 509 qdf_export_symbol(prealloc_disabled); 510 511 int qdf_mem_malloc_flags(void) 512 { 513 if (in_interrupt() || irqs_disabled() || in_atomic()) 514 return GFP_ATOMIC; 515 516 return GFP_KERNEL; 517 } 518 519 qdf_export_symbol(qdf_mem_malloc_flags); 520 521 /** 522 * qdf_prealloc_disabled_config_get() - Get the user configuration of 523 * prealloc_disabled 524 * 525 * Return: value of prealloc_disabled qdf module argument 526 */ 527 bool qdf_prealloc_disabled_config_get(void) 528 { 529 return prealloc_disabled; 530 } 531 532 qdf_export_symbol(qdf_prealloc_disabled_config_get); 533 534 #ifdef QCA_WIFI_MODULE_PARAMS_FROM_INI 535 /** 536 * qdf_prealloc_disabled_config_set() - Set prealloc_disabled 537 * @str_value: value of the module param 538 * 539 * This function will set qdf module param prealloc_disabled 540 * 541 * Return: QDF_STATUS_SUCCESS on Success 542 */ 543 QDF_STATUS qdf_prealloc_disabled_config_set(const char *str_value) 544 { 545 QDF_STATUS status; 546 547 status = qdf_bool_parse(str_value, &prealloc_disabled); 548 return status; 549 } 550 #endif 551 552 #if defined WLAN_DEBUGFS 553 554 /* Debugfs root directory for qdf_mem */ 555 static struct dentry *qdf_mem_debugfs_root; 556 557 #ifdef MEMORY_DEBUG 558 static int seq_printf_printer(void *priv, const char *fmt, ...) 559 { 560 struct seq_file *file = priv; 561 va_list args; 562 563 va_start(args, fmt); 564 seq_vprintf(file, fmt, args); 565 seq_puts(file, "\n"); 566 va_end(args); 567 568 return 0; 569 } 570 571 /** 572 * qdf_print_major_alloc() - memory metadata table print logic 573 * @table: the memory metadata table to print 574 * @print: the print adapter function 575 * @print_priv: the private data to be consumed by @print 576 * @threshold: the threshold value set by uset to list top allocations 577 * 578 * Return: None 579 */ 580 static void qdf_print_major_alloc(struct __qdf_mem_info *table, 581 qdf_abstract_print print, 582 void *print_priv, 583 uint32_t threshold) 584 { 585 int i; 586 587 for (i = 0; i < QDF_MEM_STAT_TABLE_SIZE; i++) { 588 if (!table[i].count) 589 break; 590 if (table[i].count >= threshold) 591 print(print_priv, 592 "%6u x %5u = %7uB @ %s:%u %pS %llu", 593 table[i].count, 594 table[i].size, 595 table[i].count * table[i].size, 596 table[i].func, 597 table[i].line, table[i].caller, 598 table[i].time); 599 } 600 } 601 602 /** 603 * qdf_mem_seq_start() - sequential callback to start 604 * @seq: seq_file handle 605 * @pos: The start position of the sequence 606 * 607 * Return: iterator pointer, or NULL if iteration is complete 608 */ 609 static void *qdf_mem_seq_start(struct seq_file *seq, loff_t *pos) 610 { 611 enum qdf_debug_domain domain = *pos; 612 613 if (!qdf_debug_domain_valid(domain)) 614 return NULL; 615 616 /* just use the current position as our iterator */ 617 return pos; 618 } 619 620 /** 621 * qdf_mem_seq_next() - next sequential callback 622 * @seq: seq_file handle 623 * @v: the current iterator 624 * @pos: the current position 625 * 626 * Get the next node and release previous node. 627 * 628 * Return: iterator pointer, or NULL if iteration is complete 629 */ 630 static void *qdf_mem_seq_next(struct seq_file *seq, void *v, loff_t *pos) 631 { 632 ++*pos; 633 634 return qdf_mem_seq_start(seq, pos); 635 } 636 637 /** 638 * qdf_mem_seq_stop() - stop sequential callback 639 * @seq: seq_file handle 640 * @v: current iterator 641 * 642 * Return: None 643 */ 644 static void qdf_mem_seq_stop(struct seq_file *seq, void *v) { } 645 646 /** 647 * qdf_mem_seq_show() - print sequential callback 648 * @seq: seq_file handle 649 * @v: current iterator 650 * 651 * Return: 0 - success 652 */ 653 static int qdf_mem_seq_show(struct seq_file *seq, void *v) 654 { 655 enum qdf_debug_domain domain_id = *(enum qdf_debug_domain *)v; 656 657 seq_printf(seq, "\n%s Memory Domain (Id %d)\n", 658 qdf_debug_domain_name(domain_id), domain_id); 659 qdf_mem_domain_print(qdf_mem_list_get(domain_id), 660 seq_printf_printer, 661 seq, 662 0, 663 qdf_mem_meta_table_print); 664 665 return 0; 666 } 667 668 /* sequential file operation table */ 669 static const struct seq_operations qdf_mem_seq_ops = { 670 .start = qdf_mem_seq_start, 671 .next = qdf_mem_seq_next, 672 .stop = qdf_mem_seq_stop, 673 .show = qdf_mem_seq_show, 674 }; 675 676 677 static int qdf_mem_debugfs_open(struct inode *inode, struct file *file) 678 { 679 return seq_open(file, &qdf_mem_seq_ops); 680 } 681 682 /** 683 * qdf_major_alloc_show() - print sequential callback 684 * @seq: seq_file handle 685 * @v: current iterator 686 * 687 * Return: 0 - success 688 */ 689 static int qdf_major_alloc_show(struct seq_file *seq, void *v) 690 { 691 enum qdf_debug_domain domain_id = *(enum qdf_debug_domain *)v; 692 struct major_alloc_priv *priv; 693 qdf_list_t *list; 694 695 priv = (struct major_alloc_priv *)seq->private; 696 seq_printf(seq, "\n%s Memory Domain (Id %d)\n", 697 qdf_debug_domain_name(domain_id), domain_id); 698 699 switch (priv->type) { 700 case LIST_TYPE_MEM: 701 list = qdf_mem_list_get(domain_id); 702 break; 703 case LIST_TYPE_DMA: 704 list = qdf_mem_dma_list(domain_id); 705 break; 706 default: 707 list = NULL; 708 break; 709 } 710 711 if (list) 712 qdf_mem_domain_print(list, 713 seq_printf_printer, 714 seq, 715 priv->threshold, 716 qdf_print_major_alloc); 717 718 return 0; 719 } 720 721 /* sequential file operation table created to track major allocs */ 722 static const struct seq_operations qdf_major_allocs_seq_ops = { 723 .start = qdf_mem_seq_start, 724 .next = qdf_mem_seq_next, 725 .stop = qdf_mem_seq_stop, 726 .show = qdf_major_alloc_show, 727 }; 728 729 static int qdf_major_allocs_open(struct inode *inode, struct file *file) 730 { 731 void *private = inode->i_private; 732 struct seq_file *seq; 733 int rc; 734 735 rc = seq_open(file, &qdf_major_allocs_seq_ops); 736 if (rc == 0) { 737 seq = file->private_data; 738 seq->private = private; 739 } 740 return rc; 741 } 742 743 static ssize_t qdf_major_alloc_set_threshold(struct file *file, 744 const char __user *user_buf, 745 size_t count, 746 loff_t *pos) 747 { 748 char buf[32]; 749 ssize_t buf_size; 750 uint32_t threshold; 751 struct seq_file *seq = file->private_data; 752 struct major_alloc_priv *priv = (struct major_alloc_priv *)seq->private; 753 754 buf_size = min(count, (sizeof(buf) - 1)); 755 if (buf_size <= 0) 756 return 0; 757 if (copy_from_user(buf, user_buf, buf_size)) 758 return -EFAULT; 759 buf[buf_size] = '\0'; 760 if (!kstrtou32(buf, 10, &threshold)) 761 priv->threshold = threshold; 762 return buf_size; 763 } 764 765 /** 766 * qdf_print_major_nbuf_allocs() - output agnostic nbuf print logic 767 * @threshold: the threshold value set by uset to list top allocations 768 * @print: the print adapter function 769 * @print_priv: the private data to be consumed by @print 770 * @mem_print: pointer to function which prints the memory allocation data 771 * 772 * Return: None 773 */ 774 static void 775 qdf_print_major_nbuf_allocs(uint32_t threshold, 776 qdf_abstract_print print, 777 void *print_priv, 778 void (*mem_print)(struct __qdf_mem_info *, 779 qdf_abstract_print, 780 void *, uint32_t)) 781 { 782 uint32_t nbuf_iter; 783 unsigned long irq_flag = 0; 784 QDF_NBUF_TRACK *p_node; 785 QDF_NBUF_TRACK *p_prev; 786 struct __qdf_mem_info table[QDF_MEM_STAT_TABLE_SIZE]; 787 struct qdf_mem_header meta; 788 bool is_full; 789 790 qdf_mem_zero(table, sizeof(table)); 791 qdf_mem_debug_print_header(print, print_priv, threshold); 792 793 if (is_initial_mem_debug_disabled) 794 return; 795 796 qdf_rl_info("major nbuf print with threshold %u", threshold); 797 798 for (nbuf_iter = 0; nbuf_iter < QDF_NET_BUF_TRACK_MAX_SIZE; 799 nbuf_iter++) { 800 qdf_nbuf_acquire_track_lock(nbuf_iter, irq_flag); 801 p_node = qdf_nbuf_get_track_tbl(nbuf_iter); 802 while (p_node) { 803 meta.line = p_node->line_num; 804 meta.size = p_node->size; 805 meta.caller = NULL; 806 meta.time = p_node->time; 807 qdf_str_lcopy(meta.func, p_node->func_name, 808 QDF_MEM_FUNC_NAME_SIZE); 809 810 is_full = qdf_mem_meta_table_insert(table, &meta); 811 812 if (is_full) { 813 (*mem_print)(table, print, 814 print_priv, threshold); 815 qdf_mem_zero(table, sizeof(table)); 816 } 817 818 p_prev = p_node; 819 p_node = p_node->p_next; 820 } 821 qdf_nbuf_release_track_lock(nbuf_iter, irq_flag); 822 } 823 824 (*mem_print)(table, print, print_priv, threshold); 825 826 qdf_rl_info("major nbuf print end"); 827 } 828 829 /** 830 * qdf_major_nbuf_alloc_show() - print sequential callback 831 * @seq: seq_file handle 832 * @v: current iterator 833 * 834 * Return: 0 - success 835 */ 836 static int qdf_major_nbuf_alloc_show(struct seq_file *seq, void *v) 837 { 838 struct major_alloc_priv *priv = (struct major_alloc_priv *)seq->private; 839 840 if (!priv) { 841 qdf_err("priv is null"); 842 return -EINVAL; 843 } 844 845 qdf_print_major_nbuf_allocs(priv->threshold, 846 seq_printf_printer, 847 seq, 848 qdf_print_major_alloc); 849 850 return 0; 851 } 852 853 /** 854 * qdf_nbuf_seq_start() - sequential callback to start 855 * @seq: seq_file handle 856 * @pos: The start position of the sequence 857 * 858 * Return: iterator pointer, or NULL if iteration is complete 859 */ 860 static void *qdf_nbuf_seq_start(struct seq_file *seq, loff_t *pos) 861 { 862 enum qdf_debug_domain domain = *pos; 863 864 if (domain > QDF_DEBUG_NBUF_DOMAIN) 865 return NULL; 866 867 return pos; 868 } 869 870 /** 871 * qdf_nbuf_seq_next() - next sequential callback 872 * @seq: seq_file handle 873 * @v: the current iterator 874 * @pos: the current position 875 * 876 * Get the next node and release previous node. 877 * 878 * Return: iterator pointer, or NULL if iteration is complete 879 */ 880 static void *qdf_nbuf_seq_next(struct seq_file *seq, void *v, loff_t *pos) 881 { 882 ++*pos; 883 884 return qdf_nbuf_seq_start(seq, pos); 885 } 886 887 /** 888 * qdf_nbuf_seq_stop() - stop sequential callback 889 * @seq: seq_file handle 890 * @v: current iterator 891 * 892 * Return: None 893 */ 894 static void qdf_nbuf_seq_stop(struct seq_file *seq, void *v) { } 895 896 /* sequential file operation table created to track major skb allocs */ 897 static const struct seq_operations qdf_major_nbuf_allocs_seq_ops = { 898 .start = qdf_nbuf_seq_start, 899 .next = qdf_nbuf_seq_next, 900 .stop = qdf_nbuf_seq_stop, 901 .show = qdf_major_nbuf_alloc_show, 902 }; 903 904 static int qdf_major_nbuf_allocs_open(struct inode *inode, struct file *file) 905 { 906 void *private = inode->i_private; 907 struct seq_file *seq; 908 int rc; 909 910 rc = seq_open(file, &qdf_major_nbuf_allocs_seq_ops); 911 if (rc == 0) { 912 seq = file->private_data; 913 seq->private = private; 914 } 915 return rc; 916 } 917 918 static ssize_t qdf_major_nbuf_alloc_set_threshold(struct file *file, 919 const char __user *user_buf, 920 size_t count, 921 loff_t *pos) 922 { 923 char buf[32]; 924 ssize_t buf_size; 925 uint32_t threshold; 926 struct seq_file *seq = file->private_data; 927 struct major_alloc_priv *priv = (struct major_alloc_priv *)seq->private; 928 929 buf_size = min(count, (sizeof(buf) - 1)); 930 if (buf_size <= 0) 931 return 0; 932 if (copy_from_user(buf, user_buf, buf_size)) 933 return -EFAULT; 934 buf[buf_size] = '\0'; 935 if (!kstrtou32(buf, 10, &threshold)) 936 priv->threshold = threshold; 937 return buf_size; 938 } 939 940 /* file operation table for listing major allocs */ 941 static const struct file_operations fops_qdf_major_allocs = { 942 .owner = THIS_MODULE, 943 .open = qdf_major_allocs_open, 944 .read = seq_read, 945 .llseek = seq_lseek, 946 .release = seq_release, 947 .write = qdf_major_alloc_set_threshold, 948 }; 949 950 /* debugfs file operation table */ 951 static const struct file_operations fops_qdf_mem_debugfs = { 952 .owner = THIS_MODULE, 953 .open = qdf_mem_debugfs_open, 954 .read = seq_read, 955 .llseek = seq_lseek, 956 .release = seq_release, 957 }; 958 959 /* file operation table for listing major allocs */ 960 static const struct file_operations fops_qdf_nbuf_major_allocs = { 961 .owner = THIS_MODULE, 962 .open = qdf_major_nbuf_allocs_open, 963 .read = seq_read, 964 .llseek = seq_lseek, 965 .release = seq_release, 966 .write = qdf_major_nbuf_alloc_set_threshold, 967 }; 968 969 static struct major_alloc_priv mem_priv = { 970 /* List type set to mem */ 971 LIST_TYPE_MEM, 972 /* initial threshold to list APIs which allocates mem >= 50 times */ 973 50 974 }; 975 976 static struct major_alloc_priv dma_priv = { 977 /* List type set to DMA */ 978 LIST_TYPE_DMA, 979 /* initial threshold to list APIs which allocates dma >= 50 times */ 980 50 981 }; 982 983 static struct major_alloc_priv nbuf_priv = { 984 /* List type set to NBUF */ 985 LIST_TYPE_NBUF, 986 /* initial threshold to list APIs which allocates nbuf >= 50 times */ 987 50 988 }; 989 990 static QDF_STATUS qdf_mem_debug_debugfs_init(void) 991 { 992 if (is_initial_mem_debug_disabled) 993 return QDF_STATUS_SUCCESS; 994 995 if (!qdf_mem_debugfs_root) 996 return QDF_STATUS_E_FAILURE; 997 998 debugfs_create_file("list", 999 S_IRUSR, 1000 qdf_mem_debugfs_root, 1001 NULL, 1002 &fops_qdf_mem_debugfs); 1003 1004 debugfs_create_file("major_mem_allocs", 1005 0600, 1006 qdf_mem_debugfs_root, 1007 &mem_priv, 1008 &fops_qdf_major_allocs); 1009 1010 debugfs_create_file("major_dma_allocs", 1011 0600, 1012 qdf_mem_debugfs_root, 1013 &dma_priv, 1014 &fops_qdf_major_allocs); 1015 1016 debugfs_create_file("major_nbuf_allocs", 1017 0600, 1018 qdf_mem_debugfs_root, 1019 &nbuf_priv, 1020 &fops_qdf_nbuf_major_allocs); 1021 1022 return QDF_STATUS_SUCCESS; 1023 } 1024 1025 static QDF_STATUS qdf_mem_debug_debugfs_exit(void) 1026 { 1027 return QDF_STATUS_SUCCESS; 1028 } 1029 1030 #else /* MEMORY_DEBUG */ 1031 1032 static QDF_STATUS qdf_mem_debug_debugfs_init(void) 1033 { 1034 return QDF_STATUS_E_NOSUPPORT; 1035 } 1036 1037 static QDF_STATUS qdf_mem_debug_debugfs_exit(void) 1038 { 1039 return QDF_STATUS_E_NOSUPPORT; 1040 } 1041 1042 #endif /* MEMORY_DEBUG */ 1043 1044 1045 static void qdf_mem_debugfs_exit(void) 1046 { 1047 debugfs_remove_recursive(qdf_mem_debugfs_root); 1048 qdf_mem_debugfs_root = NULL; 1049 } 1050 1051 static QDF_STATUS qdf_mem_debugfs_init(void) 1052 { 1053 struct dentry *qdf_debugfs_root = qdf_debugfs_get_root(); 1054 1055 if (!qdf_debugfs_root) 1056 return QDF_STATUS_E_FAILURE; 1057 1058 qdf_mem_debugfs_root = debugfs_create_dir("mem", qdf_debugfs_root); 1059 1060 if (!qdf_mem_debugfs_root) 1061 return QDF_STATUS_E_FAILURE; 1062 1063 1064 debugfs_create_atomic_t("kmalloc", 1065 S_IRUSR, 1066 qdf_mem_debugfs_root, 1067 &qdf_mem_stat.kmalloc); 1068 1069 debugfs_create_atomic_t("dma", 1070 S_IRUSR, 1071 qdf_mem_debugfs_root, 1072 &qdf_mem_stat.dma); 1073 1074 debugfs_create_atomic_t("skb", 1075 S_IRUSR, 1076 qdf_mem_debugfs_root, 1077 &qdf_mem_stat.skb); 1078 1079 return QDF_STATUS_SUCCESS; 1080 } 1081 1082 #else /* WLAN_DEBUGFS */ 1083 1084 static QDF_STATUS qdf_mem_debugfs_init(void) 1085 { 1086 return QDF_STATUS_E_NOSUPPORT; 1087 } 1088 static void qdf_mem_debugfs_exit(void) {} 1089 1090 1091 static QDF_STATUS qdf_mem_debug_debugfs_init(void) 1092 { 1093 return QDF_STATUS_E_NOSUPPORT; 1094 } 1095 1096 static QDF_STATUS qdf_mem_debug_debugfs_exit(void) 1097 { 1098 return QDF_STATUS_E_NOSUPPORT; 1099 } 1100 1101 #endif /* WLAN_DEBUGFS */ 1102 1103 void qdf_mem_kmalloc_inc(qdf_size_t size) 1104 { 1105 qdf_atomic_add(size, &qdf_mem_stat.kmalloc); 1106 } 1107 1108 static void qdf_mem_dma_inc(qdf_size_t size) 1109 { 1110 qdf_atomic_add(size, &qdf_mem_stat.dma); 1111 } 1112 1113 #ifdef CONFIG_WLAN_SYSFS_MEM_STATS 1114 void qdf_mem_skb_inc(qdf_size_t size) 1115 { 1116 qdf_atomic_add(size, &qdf_mem_stat.skb); 1117 } 1118 1119 void qdf_mem_skb_dec(qdf_size_t size) 1120 { 1121 qdf_atomic_sub(size, &qdf_mem_stat.skb); 1122 } 1123 1124 void qdf_mem_skb_total_inc(qdf_size_t size) 1125 { 1126 int32_t skb_mem_max = 0; 1127 1128 qdf_atomic_add(size, &qdf_mem_stat.skb_total); 1129 skb_mem_max = qdf_atomic_read(&qdf_mem_stat.skb_total); 1130 if (qdf_mem_stat.skb_mem_max < skb_mem_max) 1131 qdf_mem_stat.skb_mem_max = skb_mem_max; 1132 } 1133 1134 void qdf_mem_skb_total_dec(qdf_size_t size) 1135 { 1136 qdf_atomic_sub(size, &qdf_mem_stat.skb_total); 1137 } 1138 1139 void qdf_mem_dp_tx_skb_inc(qdf_size_t size) 1140 { 1141 int32_t curr_dp_tx_skb_mem_max = 0; 1142 1143 qdf_atomic_add(size, &qdf_mem_stat.dp_tx_skb); 1144 curr_dp_tx_skb_mem_max = qdf_atomic_read(&qdf_mem_stat.dp_tx_skb); 1145 if (qdf_mem_stat.dp_tx_skb_mem_max < curr_dp_tx_skb_mem_max) 1146 qdf_mem_stat.dp_tx_skb_mem_max = curr_dp_tx_skb_mem_max; 1147 } 1148 1149 void qdf_mem_dp_tx_skb_dec(qdf_size_t size) 1150 { 1151 qdf_atomic_sub(size, &qdf_mem_stat.dp_tx_skb); 1152 } 1153 1154 void qdf_mem_dp_rx_skb_inc(qdf_size_t size) 1155 { 1156 int32_t curr_dp_rx_skb_mem_max = 0; 1157 1158 qdf_atomic_add(size, &qdf_mem_stat.dp_rx_skb); 1159 curr_dp_rx_skb_mem_max = qdf_atomic_read(&qdf_mem_stat.dp_rx_skb); 1160 if (qdf_mem_stat.dp_rx_skb_mem_max < curr_dp_rx_skb_mem_max) 1161 qdf_mem_stat.dp_rx_skb_mem_max = curr_dp_rx_skb_mem_max; 1162 } 1163 1164 void qdf_mem_dp_rx_skb_dec(qdf_size_t size) 1165 { 1166 qdf_atomic_sub(size, &qdf_mem_stat.dp_rx_skb); 1167 } 1168 1169 void qdf_mem_dp_tx_skb_cnt_inc(void) 1170 { 1171 int32_t curr_dp_tx_skb_count_max = 0; 1172 1173 qdf_atomic_add(1, &qdf_mem_stat.dp_tx_skb_count); 1174 curr_dp_tx_skb_count_max = 1175 qdf_atomic_read(&qdf_mem_stat.dp_tx_skb_count); 1176 if (qdf_mem_stat.dp_tx_skb_count_max < curr_dp_tx_skb_count_max) 1177 qdf_mem_stat.dp_tx_skb_count_max = curr_dp_tx_skb_count_max; 1178 } 1179 1180 void qdf_mem_dp_tx_skb_cnt_dec(void) 1181 { 1182 qdf_atomic_sub(1, &qdf_mem_stat.dp_tx_skb_count); 1183 } 1184 1185 void qdf_mem_dp_rx_skb_cnt_inc(void) 1186 { 1187 int32_t curr_dp_rx_skb_count_max = 0; 1188 1189 qdf_atomic_add(1, &qdf_mem_stat.dp_rx_skb_count); 1190 curr_dp_rx_skb_count_max = 1191 qdf_atomic_read(&qdf_mem_stat.dp_rx_skb_count); 1192 if (qdf_mem_stat.dp_rx_skb_count_max < curr_dp_rx_skb_count_max) 1193 qdf_mem_stat.dp_rx_skb_count_max = curr_dp_rx_skb_count_max; 1194 } 1195 1196 void qdf_mem_dp_rx_skb_cnt_dec(void) 1197 { 1198 qdf_atomic_sub(1, &qdf_mem_stat.dp_rx_skb_count); 1199 } 1200 #endif 1201 1202 void qdf_mem_kmalloc_dec(qdf_size_t size) 1203 { 1204 qdf_atomic_sub(size, &qdf_mem_stat.kmalloc); 1205 } 1206 1207 static inline void qdf_mem_dma_dec(qdf_size_t size) 1208 { 1209 qdf_atomic_sub(size, &qdf_mem_stat.dma); 1210 } 1211 1212 /** 1213 * __qdf_mempool_init() - Create and initialize memory pool 1214 * 1215 * @osdev: platform device object 1216 * @pool_addr: address of the pool created 1217 * @elem_cnt: no. of elements in pool 1218 * @elem_size: size of each pool element in bytes 1219 * @flags: flags 1220 * 1221 * return: Handle to memory pool or NULL if allocation failed 1222 */ 1223 int __qdf_mempool_init(qdf_device_t osdev, __qdf_mempool_t *pool_addr, 1224 int elem_cnt, size_t elem_size, u_int32_t flags) 1225 { 1226 __qdf_mempool_ctxt_t *new_pool = NULL; 1227 u_int32_t align = L1_CACHE_BYTES; 1228 unsigned long aligned_pool_mem; 1229 int pool_id; 1230 int i; 1231 1232 if (prealloc_disabled) { 1233 /* TBD: We can maintain a list of pools in qdf_device_t 1234 * to help debugging 1235 * when pre-allocation is not enabled 1236 */ 1237 new_pool = (__qdf_mempool_ctxt_t *) 1238 kmalloc(sizeof(__qdf_mempool_ctxt_t), GFP_KERNEL); 1239 if (!new_pool) 1240 return QDF_STATUS_E_NOMEM; 1241 1242 memset(new_pool, 0, sizeof(*new_pool)); 1243 /* TBD: define flags for zeroing buffers etc */ 1244 new_pool->flags = flags; 1245 new_pool->elem_size = elem_size; 1246 new_pool->max_elem = elem_cnt; 1247 *pool_addr = new_pool; 1248 return 0; 1249 } 1250 1251 for (pool_id = 0; pool_id < MAX_MEM_POOLS; pool_id++) { 1252 if (!osdev->mem_pool[pool_id]) 1253 break; 1254 } 1255 1256 if (pool_id == MAX_MEM_POOLS) 1257 return -ENOMEM; 1258 1259 new_pool = osdev->mem_pool[pool_id] = (__qdf_mempool_ctxt_t *) 1260 kmalloc(sizeof(__qdf_mempool_ctxt_t), GFP_KERNEL); 1261 if (!new_pool) 1262 return -ENOMEM; 1263 1264 memset(new_pool, 0, sizeof(*new_pool)); 1265 /* TBD: define flags for zeroing buffers etc */ 1266 new_pool->flags = flags; 1267 new_pool->pool_id = pool_id; 1268 1269 /* Round up the element size to cacheline */ 1270 new_pool->elem_size = roundup(elem_size, L1_CACHE_BYTES); 1271 new_pool->mem_size = elem_cnt * new_pool->elem_size + 1272 ((align)?(align - 1):0); 1273 1274 new_pool->pool_mem = kzalloc(new_pool->mem_size, GFP_KERNEL); 1275 if (!new_pool->pool_mem) { 1276 /* TBD: Check if we need get_free_pages above */ 1277 kfree(new_pool); 1278 osdev->mem_pool[pool_id] = NULL; 1279 return -ENOMEM; 1280 } 1281 1282 spin_lock_init(&new_pool->lock); 1283 1284 /* Initialize free list */ 1285 aligned_pool_mem = (unsigned long)(new_pool->pool_mem) + 1286 ((align) ? (unsigned long)(new_pool->pool_mem)%align:0); 1287 STAILQ_INIT(&new_pool->free_list); 1288 1289 for (i = 0; i < elem_cnt; i++) 1290 STAILQ_INSERT_TAIL(&(new_pool->free_list), 1291 (mempool_elem_t *)(aligned_pool_mem + 1292 (new_pool->elem_size * i)), mempool_entry); 1293 1294 1295 new_pool->free_cnt = elem_cnt; 1296 *pool_addr = new_pool; 1297 return 0; 1298 } 1299 qdf_export_symbol(__qdf_mempool_init); 1300 1301 /** 1302 * __qdf_mempool_destroy() - Destroy memory pool 1303 * @osdev: platform device object 1304 * @Handle: to memory pool 1305 * 1306 * Returns: none 1307 */ 1308 void __qdf_mempool_destroy(qdf_device_t osdev, __qdf_mempool_t pool) 1309 { 1310 int pool_id = 0; 1311 1312 if (!pool) 1313 return; 1314 1315 if (prealloc_disabled) { 1316 kfree(pool); 1317 return; 1318 } 1319 1320 pool_id = pool->pool_id; 1321 1322 /* TBD: Check if free count matches elem_cnt if debug is enabled */ 1323 kfree(pool->pool_mem); 1324 kfree(pool); 1325 osdev->mem_pool[pool_id] = NULL; 1326 } 1327 qdf_export_symbol(__qdf_mempool_destroy); 1328 1329 /** 1330 * __qdf_mempool_alloc() - Allocate an element memory pool 1331 * 1332 * @osdev: platform device object 1333 * @Handle: to memory pool 1334 * 1335 * Return: Pointer to the allocated element or NULL if the pool is empty 1336 */ 1337 void *__qdf_mempool_alloc(qdf_device_t osdev, __qdf_mempool_t pool) 1338 { 1339 void *buf = NULL; 1340 1341 if (!pool) 1342 return NULL; 1343 1344 if (prealloc_disabled) 1345 return qdf_mem_malloc(pool->elem_size); 1346 1347 spin_lock_bh(&pool->lock); 1348 1349 buf = STAILQ_FIRST(&pool->free_list); 1350 if (buf) { 1351 STAILQ_REMOVE_HEAD(&pool->free_list, mempool_entry); 1352 pool->free_cnt--; 1353 } 1354 1355 /* TBD: Update free count if debug is enabled */ 1356 spin_unlock_bh(&pool->lock); 1357 1358 return buf; 1359 } 1360 qdf_export_symbol(__qdf_mempool_alloc); 1361 1362 /** 1363 * __qdf_mempool_free() - Free a memory pool element 1364 * @osdev: Platform device object 1365 * @pool: Handle to memory pool 1366 * @buf: Element to be freed 1367 * 1368 * Returns: none 1369 */ 1370 void __qdf_mempool_free(qdf_device_t osdev, __qdf_mempool_t pool, void *buf) 1371 { 1372 if (!pool) 1373 return; 1374 1375 1376 if (prealloc_disabled) 1377 return qdf_mem_free(buf); 1378 1379 spin_lock_bh(&pool->lock); 1380 pool->free_cnt++; 1381 1382 STAILQ_INSERT_TAIL 1383 (&pool->free_list, (mempool_elem_t *)buf, mempool_entry); 1384 spin_unlock_bh(&pool->lock); 1385 } 1386 qdf_export_symbol(__qdf_mempool_free); 1387 1388 #ifdef CNSS_MEM_PRE_ALLOC 1389 static bool qdf_might_be_prealloc(void *ptr) 1390 { 1391 if (ksize(ptr) > WCNSS_PRE_ALLOC_GET_THRESHOLD) 1392 return true; 1393 else 1394 return false; 1395 } 1396 1397 /** 1398 * qdf_mem_prealloc_get() - conditionally pre-allocate memory 1399 * @size: the number of bytes to allocate 1400 * 1401 * If size if greater than WCNSS_PRE_ALLOC_GET_THRESHOLD, this function returns 1402 * a chunk of pre-allocated memory. If size if less than or equal to 1403 * WCNSS_PRE_ALLOC_GET_THRESHOLD, or an error occurs, NULL is returned instead. 1404 * 1405 * Return: NULL on failure, non-NULL on success 1406 */ 1407 static void *qdf_mem_prealloc_get(size_t size) 1408 { 1409 void *ptr; 1410 1411 if (size <= WCNSS_PRE_ALLOC_GET_THRESHOLD) 1412 return NULL; 1413 1414 ptr = wcnss_prealloc_get(size); 1415 if (!ptr) 1416 return NULL; 1417 1418 memset(ptr, 0, size); 1419 1420 return ptr; 1421 } 1422 1423 static inline bool qdf_mem_prealloc_put(void *ptr) 1424 { 1425 return wcnss_prealloc_put(ptr); 1426 } 1427 #else 1428 static bool qdf_might_be_prealloc(void *ptr) 1429 { 1430 return false; 1431 } 1432 1433 static inline void *qdf_mem_prealloc_get(size_t size) 1434 { 1435 return NULL; 1436 } 1437 1438 static inline bool qdf_mem_prealloc_put(void *ptr) 1439 { 1440 return false; 1441 } 1442 #endif /* CNSS_MEM_PRE_ALLOC */ 1443 1444 /* External Function implementation */ 1445 #ifdef MEMORY_DEBUG 1446 /** 1447 * qdf_mem_debug_config_get() - Get the user configuration of mem_debug_disabled 1448 * 1449 * Return: value of mem_debug_disabled qdf module argument 1450 */ 1451 #ifdef DISABLE_MEM_DBG_LOAD_CONFIG 1452 bool qdf_mem_debug_config_get(void) 1453 { 1454 /* Return false if DISABLE_LOAD_MEM_DBG_CONFIG flag is enabled */ 1455 return false; 1456 } 1457 #else 1458 bool qdf_mem_debug_config_get(void) 1459 { 1460 return mem_debug_disabled; 1461 } 1462 #endif /* DISABLE_MEM_DBG_LOAD_CONFIG */ 1463 1464 /** 1465 * qdf_mem_debug_disabled_set() - Set mem_debug_disabled 1466 * @str_value: value of the module param 1467 * 1468 * This function will se qdf module param mem_debug_disabled 1469 * 1470 * Return: QDF_STATUS_SUCCESS on Success 1471 */ 1472 #ifdef QCA_WIFI_MODULE_PARAMS_FROM_INI 1473 QDF_STATUS qdf_mem_debug_disabled_config_set(const char *str_value) 1474 { 1475 QDF_STATUS status; 1476 1477 status = qdf_bool_parse(str_value, &mem_debug_disabled); 1478 return status; 1479 } 1480 #endif 1481 1482 /** 1483 * qdf_mem_debug_init() - initialize qdf memory debug functionality 1484 * 1485 * Return: none 1486 */ 1487 static void qdf_mem_debug_init(void) 1488 { 1489 int i; 1490 1491 is_initial_mem_debug_disabled = qdf_mem_debug_config_get(); 1492 1493 if (is_initial_mem_debug_disabled) 1494 return; 1495 1496 /* Initalizing the list with maximum size of 60000 */ 1497 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 1498 qdf_list_create(&qdf_mem_domains[i], 60000); 1499 qdf_spinlock_create(&qdf_mem_list_lock); 1500 1501 /* dma */ 1502 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 1503 qdf_list_create(&qdf_mem_dma_domains[i], 0); 1504 qdf_spinlock_create(&qdf_mem_dma_list_lock); 1505 } 1506 1507 static uint32_t 1508 qdf_mem_domain_check_for_leaks(enum qdf_debug_domain domain, 1509 qdf_list_t *mem_list) 1510 { 1511 if (is_initial_mem_debug_disabled) 1512 return 0; 1513 1514 if (qdf_list_empty(mem_list)) 1515 return 0; 1516 1517 qdf_err("Memory leaks detected in %s domain!", 1518 qdf_debug_domain_name(domain)); 1519 qdf_mem_domain_print(mem_list, 1520 qdf_err_printer, 1521 NULL, 1522 0, 1523 qdf_mem_meta_table_print); 1524 1525 return mem_list->count; 1526 } 1527 1528 static void qdf_mem_domain_set_check_for_leaks(qdf_list_t *domains) 1529 { 1530 uint32_t leak_count = 0; 1531 int i; 1532 1533 if (is_initial_mem_debug_disabled) 1534 return; 1535 1536 /* detect and print leaks */ 1537 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 1538 leak_count += qdf_mem_domain_check_for_leaks(i, domains + i); 1539 1540 if (leak_count) 1541 QDF_MEMDEBUG_PANIC("%u fatal memory leaks detected!", 1542 leak_count); 1543 } 1544 1545 /** 1546 * qdf_mem_debug_exit() - exit qdf memory debug functionality 1547 * 1548 * Return: none 1549 */ 1550 static void qdf_mem_debug_exit(void) 1551 { 1552 int i; 1553 1554 if (is_initial_mem_debug_disabled) 1555 return; 1556 1557 /* mem */ 1558 qdf_mem_domain_set_check_for_leaks(qdf_mem_domains); 1559 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 1560 qdf_list_destroy(qdf_mem_list_get(i)); 1561 1562 qdf_spinlock_destroy(&qdf_mem_list_lock); 1563 1564 /* dma */ 1565 qdf_mem_domain_set_check_for_leaks(qdf_mem_dma_domains); 1566 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 1567 qdf_list_destroy(&qdf_mem_dma_domains[i]); 1568 qdf_spinlock_destroy(&qdf_mem_dma_list_lock); 1569 } 1570 1571 void *qdf_mem_malloc_debug(size_t size, const char *func, uint32_t line, 1572 void *caller, uint32_t flag) 1573 { 1574 QDF_STATUS status; 1575 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1576 qdf_list_t *mem_list = qdf_mem_list_get(current_domain); 1577 struct qdf_mem_header *header; 1578 void *ptr; 1579 unsigned long start, duration; 1580 1581 if (is_initial_mem_debug_disabled) 1582 return __qdf_mem_malloc(size, func, line); 1583 1584 if (!size || size > QDF_MEM_MAX_MALLOC) { 1585 qdf_err("Cannot malloc %zu bytes @ %s:%d", size, func, line); 1586 return NULL; 1587 } 1588 1589 ptr = qdf_mem_prealloc_get(size); 1590 if (ptr) 1591 return ptr; 1592 1593 if (!flag) 1594 flag = qdf_mem_malloc_flags(); 1595 1596 start = qdf_mc_timer_get_system_time(); 1597 header = kzalloc(size + QDF_MEM_DEBUG_SIZE, flag); 1598 duration = qdf_mc_timer_get_system_time() - start; 1599 1600 if (duration > QDF_MEM_WARN_THRESHOLD) 1601 qdf_warn("Malloc slept; %lums, %zuB @ %s:%d", 1602 duration, size, func, line); 1603 1604 if (!header) { 1605 qdf_warn("Failed to malloc %zuB @ %s:%d", size, func, line); 1606 return NULL; 1607 } 1608 1609 qdf_mem_header_init(header, size, func, line, caller); 1610 qdf_mem_trailer_init(header); 1611 ptr = qdf_mem_get_ptr(header); 1612 1613 qdf_spin_lock_irqsave(&qdf_mem_list_lock); 1614 status = qdf_list_insert_front(mem_list, &header->node); 1615 qdf_spin_unlock_irqrestore(&qdf_mem_list_lock); 1616 if (QDF_IS_STATUS_ERROR(status)) 1617 qdf_err("Failed to insert memory header; status %d", status); 1618 1619 qdf_mem_kmalloc_inc(ksize(header)); 1620 1621 return ptr; 1622 } 1623 qdf_export_symbol(qdf_mem_malloc_debug); 1624 1625 void *qdf_mem_malloc_atomic_debug(size_t size, const char *func, 1626 uint32_t line, void *caller) 1627 { 1628 QDF_STATUS status; 1629 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1630 qdf_list_t *mem_list = qdf_mem_list_get(current_domain); 1631 struct qdf_mem_header *header; 1632 void *ptr; 1633 unsigned long start, duration; 1634 1635 if (is_initial_mem_debug_disabled) 1636 return qdf_mem_malloc_atomic_debug_fl(size, func, line); 1637 1638 if (!size || size > QDF_MEM_MAX_MALLOC) { 1639 qdf_err("Cannot malloc %zu bytes @ %s:%d", size, func, line); 1640 return NULL; 1641 } 1642 1643 ptr = qdf_mem_prealloc_get(size); 1644 if (ptr) 1645 return ptr; 1646 1647 start = qdf_mc_timer_get_system_time(); 1648 header = kzalloc(size + QDF_MEM_DEBUG_SIZE, GFP_ATOMIC); 1649 duration = qdf_mc_timer_get_system_time() - start; 1650 1651 if (duration > QDF_MEM_WARN_THRESHOLD) 1652 qdf_warn("Malloc slept; %lums, %zuB @ %s:%d", 1653 duration, size, func, line); 1654 1655 if (!header) { 1656 qdf_warn("Failed to malloc %zuB @ %s:%d", size, func, line); 1657 return NULL; 1658 } 1659 1660 qdf_mem_header_init(header, size, func, line, caller); 1661 qdf_mem_trailer_init(header); 1662 ptr = qdf_mem_get_ptr(header); 1663 1664 qdf_spin_lock_irqsave(&qdf_mem_list_lock); 1665 status = qdf_list_insert_front(mem_list, &header->node); 1666 qdf_spin_unlock_irqrestore(&qdf_mem_list_lock); 1667 if (QDF_IS_STATUS_ERROR(status)) 1668 qdf_err("Failed to insert memory header; status %d", status); 1669 1670 qdf_mem_kmalloc_inc(ksize(header)); 1671 1672 return ptr; 1673 } 1674 1675 qdf_export_symbol(qdf_mem_malloc_atomic_debug); 1676 1677 void *qdf_mem_malloc_atomic_debug_fl(size_t size, const char *func, 1678 uint32_t line) 1679 { 1680 void *ptr; 1681 1682 if (!size || size > QDF_MEM_MAX_MALLOC) { 1683 qdf_nofl_err("Cannot malloc %zu bytes @ %s:%d", size, func, 1684 line); 1685 return NULL; 1686 } 1687 1688 ptr = qdf_mem_prealloc_get(size); 1689 if (ptr) 1690 return ptr; 1691 1692 ptr = kzalloc(size, GFP_ATOMIC); 1693 if (!ptr) { 1694 qdf_nofl_warn("Failed to malloc %zuB @ %s:%d", 1695 size, func, line); 1696 return NULL; 1697 } 1698 1699 qdf_mem_kmalloc_inc(ksize(ptr)); 1700 1701 return ptr; 1702 } 1703 1704 qdf_export_symbol(qdf_mem_malloc_atomic_debug_fl); 1705 1706 void qdf_mem_free_debug(void *ptr, const char *func, uint32_t line) 1707 { 1708 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1709 struct qdf_mem_header *header; 1710 enum qdf_mem_validation_bitmap error_bitmap; 1711 1712 if (is_initial_mem_debug_disabled) { 1713 __qdf_mem_free(ptr); 1714 return; 1715 } 1716 1717 /* freeing a null pointer is valid */ 1718 if (qdf_unlikely(!ptr)) 1719 return; 1720 1721 if (qdf_mem_prealloc_put(ptr)) 1722 return; 1723 1724 if (qdf_unlikely((qdf_size_t)ptr <= sizeof(*header))) 1725 QDF_MEMDEBUG_PANIC("Failed to free invalid memory location %pK", 1726 ptr); 1727 1728 qdf_talloc_assert_no_children_fl(ptr, func, line); 1729 1730 qdf_spin_lock_irqsave(&qdf_mem_list_lock); 1731 header = qdf_mem_get_header(ptr); 1732 error_bitmap = qdf_mem_header_validate(header, current_domain); 1733 error_bitmap |= qdf_mem_trailer_validate(header); 1734 1735 if (!error_bitmap) { 1736 header->freed = true; 1737 qdf_list_remove_node(qdf_mem_list_get(header->domain), 1738 &header->node); 1739 } 1740 qdf_spin_unlock_irqrestore(&qdf_mem_list_lock); 1741 1742 qdf_mem_header_assert_valid(header, current_domain, error_bitmap, 1743 func, line); 1744 1745 qdf_mem_kmalloc_dec(ksize(header)); 1746 kfree(header); 1747 } 1748 qdf_export_symbol(qdf_mem_free_debug); 1749 1750 void qdf_mem_check_for_leaks(void) 1751 { 1752 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1753 qdf_list_t *mem_list = qdf_mem_list_get(current_domain); 1754 qdf_list_t *dma_list = qdf_mem_dma_list(current_domain); 1755 uint32_t leaks_count = 0; 1756 1757 if (is_initial_mem_debug_disabled) 1758 return; 1759 1760 leaks_count += qdf_mem_domain_check_for_leaks(current_domain, mem_list); 1761 leaks_count += qdf_mem_domain_check_for_leaks(current_domain, dma_list); 1762 1763 if (leaks_count) 1764 QDF_MEMDEBUG_PANIC("%u fatal memory leaks detected!", 1765 leaks_count); 1766 } 1767 1768 /** 1769 * qdf_mem_multi_pages_alloc_debug() - Debug version of 1770 * qdf_mem_multi_pages_alloc 1771 * @osdev: OS device handle pointer 1772 * @pages: Multi page information storage 1773 * @element_size: Each element size 1774 * @element_num: Total number of elements should be allocated 1775 * @memctxt: Memory context 1776 * @cacheable: Coherent memory or cacheable memory 1777 * @func: Caller of this allocator 1778 * @line: Line number of the caller 1779 * @caller: Return address of the caller 1780 * 1781 * This function will allocate large size of memory over multiple pages. 1782 * Large size of contiguous memory allocation will fail frequently, then 1783 * instead of allocate large memory by one shot, allocate through multiple, non 1784 * contiguous memory and combine pages when actual usage 1785 * 1786 * Return: None 1787 */ 1788 void qdf_mem_multi_pages_alloc_debug(qdf_device_t osdev, 1789 struct qdf_mem_multi_page_t *pages, 1790 size_t element_size, uint32_t element_num, 1791 qdf_dma_context_t memctxt, bool cacheable, 1792 const char *func, uint32_t line, 1793 void *caller) 1794 { 1795 uint16_t page_idx; 1796 struct qdf_mem_dma_page_t *dma_pages; 1797 void **cacheable_pages = NULL; 1798 uint16_t i; 1799 1800 if (!pages->page_size) 1801 pages->page_size = qdf_page_size; 1802 1803 pages->num_element_per_page = pages->page_size / element_size; 1804 if (!pages->num_element_per_page) { 1805 qdf_print("Invalid page %d or element size %d", 1806 (int)pages->page_size, (int)element_size); 1807 goto out_fail; 1808 } 1809 1810 pages->num_pages = element_num / pages->num_element_per_page; 1811 if (element_num % pages->num_element_per_page) 1812 pages->num_pages++; 1813 1814 if (cacheable) { 1815 /* Pages information storage */ 1816 pages->cacheable_pages = qdf_mem_malloc_debug( 1817 pages->num_pages * sizeof(pages->cacheable_pages), 1818 func, line, caller, 0); 1819 if (!pages->cacheable_pages) 1820 goto out_fail; 1821 1822 cacheable_pages = pages->cacheable_pages; 1823 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 1824 cacheable_pages[page_idx] = qdf_mem_malloc_debug( 1825 pages->page_size, func, line, caller, 0); 1826 if (!cacheable_pages[page_idx]) 1827 goto page_alloc_fail; 1828 } 1829 pages->dma_pages = NULL; 1830 } else { 1831 pages->dma_pages = qdf_mem_malloc_debug( 1832 pages->num_pages * sizeof(struct qdf_mem_dma_page_t), 1833 func, line, caller, 0); 1834 if (!pages->dma_pages) 1835 goto out_fail; 1836 1837 dma_pages = pages->dma_pages; 1838 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 1839 dma_pages->page_v_addr_start = 1840 qdf_mem_alloc_consistent_debug( 1841 osdev, osdev->dev, pages->page_size, 1842 &dma_pages->page_p_addr, 1843 func, line, caller); 1844 if (!dma_pages->page_v_addr_start) { 1845 qdf_print("dmaable page alloc fail pi %d", 1846 page_idx); 1847 goto page_alloc_fail; 1848 } 1849 dma_pages->page_v_addr_end = 1850 dma_pages->page_v_addr_start + pages->page_size; 1851 dma_pages++; 1852 } 1853 pages->cacheable_pages = NULL; 1854 } 1855 return; 1856 1857 page_alloc_fail: 1858 if (cacheable) { 1859 for (i = 0; i < page_idx; i++) 1860 qdf_mem_free_debug(pages->cacheable_pages[i], 1861 func, line); 1862 qdf_mem_free_debug(pages->cacheable_pages, func, line); 1863 } else { 1864 dma_pages = pages->dma_pages; 1865 for (i = 0; i < page_idx; i++) { 1866 qdf_mem_free_consistent_debug( 1867 osdev, osdev->dev, 1868 pages->page_size, dma_pages->page_v_addr_start, 1869 dma_pages->page_p_addr, memctxt, func, line); 1870 dma_pages++; 1871 } 1872 qdf_mem_free_debug(pages->dma_pages, func, line); 1873 } 1874 1875 out_fail: 1876 pages->cacheable_pages = NULL; 1877 pages->dma_pages = NULL; 1878 pages->num_pages = 0; 1879 } 1880 1881 qdf_export_symbol(qdf_mem_multi_pages_alloc_debug); 1882 1883 /** 1884 * qdf_mem_multi_pages_free_debug() - Debug version of qdf_mem_multi_pages_free 1885 * @osdev: OS device handle pointer 1886 * @pages: Multi page information storage 1887 * @memctxt: Memory context 1888 * @cacheable: Coherent memory or cacheable memory 1889 * @func: Caller of this allocator 1890 * @line: Line number of the caller 1891 * 1892 * This function will free large size of memory over multiple pages. 1893 * 1894 * Return: None 1895 */ 1896 void qdf_mem_multi_pages_free_debug(qdf_device_t osdev, 1897 struct qdf_mem_multi_page_t *pages, 1898 qdf_dma_context_t memctxt, bool cacheable, 1899 const char *func, uint32_t line) 1900 { 1901 unsigned int page_idx; 1902 struct qdf_mem_dma_page_t *dma_pages; 1903 1904 if (!pages->page_size) 1905 pages->page_size = qdf_page_size; 1906 1907 if (cacheable) { 1908 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) 1909 qdf_mem_free_debug(pages->cacheable_pages[page_idx], 1910 func, line); 1911 qdf_mem_free_debug(pages->cacheable_pages, func, line); 1912 } else { 1913 dma_pages = pages->dma_pages; 1914 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 1915 qdf_mem_free_consistent_debug( 1916 osdev, osdev->dev, pages->page_size, 1917 dma_pages->page_v_addr_start, 1918 dma_pages->page_p_addr, memctxt, func, line); 1919 dma_pages++; 1920 } 1921 qdf_mem_free_debug(pages->dma_pages, func, line); 1922 } 1923 1924 pages->cacheable_pages = NULL; 1925 pages->dma_pages = NULL; 1926 pages->num_pages = 0; 1927 } 1928 1929 qdf_export_symbol(qdf_mem_multi_pages_free_debug); 1930 1931 #else 1932 static void qdf_mem_debug_init(void) {} 1933 1934 static void qdf_mem_debug_exit(void) {} 1935 1936 void *qdf_mem_malloc_atomic_fl(size_t size, const char *func, uint32_t line) 1937 { 1938 void *ptr; 1939 1940 if (!size || size > QDF_MEM_MAX_MALLOC) { 1941 qdf_nofl_err("Cannot malloc %zu bytes @ %s:%d", size, func, 1942 line); 1943 return NULL; 1944 } 1945 1946 ptr = qdf_mem_prealloc_get(size); 1947 if (ptr) 1948 return ptr; 1949 1950 ptr = kzalloc(size, GFP_ATOMIC); 1951 if (!ptr) { 1952 qdf_nofl_warn("Failed to malloc %zuB @ %s:%d", 1953 size, func, line); 1954 return NULL; 1955 } 1956 1957 qdf_mem_kmalloc_inc(ksize(ptr)); 1958 1959 return ptr; 1960 } 1961 qdf_export_symbol(qdf_mem_malloc_atomic_fl); 1962 1963 /** 1964 * qdf_mem_multi_pages_alloc() - allocate large size of kernel memory 1965 * @osdev: OS device handle pointer 1966 * @pages: Multi page information storage 1967 * @element_size: Each element size 1968 * @element_num: Total number of elements should be allocated 1969 * @memctxt: Memory context 1970 * @cacheable: Coherent memory or cacheable memory 1971 * 1972 * This function will allocate large size of memory over multiple pages. 1973 * Large size of contiguous memory allocation will fail frequently, then 1974 * instead of allocate large memory by one shot, allocate through multiple, non 1975 * contiguous memory and combine pages when actual usage 1976 * 1977 * Return: None 1978 */ 1979 void qdf_mem_multi_pages_alloc(qdf_device_t osdev, 1980 struct qdf_mem_multi_page_t *pages, 1981 size_t element_size, uint32_t element_num, 1982 qdf_dma_context_t memctxt, bool cacheable) 1983 { 1984 uint16_t page_idx; 1985 struct qdf_mem_dma_page_t *dma_pages; 1986 void **cacheable_pages = NULL; 1987 uint16_t i; 1988 1989 if (!pages->page_size) 1990 pages->page_size = qdf_page_size; 1991 1992 pages->num_element_per_page = pages->page_size / element_size; 1993 if (!pages->num_element_per_page) { 1994 qdf_print("Invalid page %d or element size %d", 1995 (int)pages->page_size, (int)element_size); 1996 goto out_fail; 1997 } 1998 1999 pages->num_pages = element_num / pages->num_element_per_page; 2000 if (element_num % pages->num_element_per_page) 2001 pages->num_pages++; 2002 2003 if (cacheable) { 2004 /* Pages information storage */ 2005 pages->cacheable_pages = qdf_mem_malloc( 2006 pages->num_pages * sizeof(pages->cacheable_pages)); 2007 if (!pages->cacheable_pages) 2008 goto out_fail; 2009 2010 cacheable_pages = pages->cacheable_pages; 2011 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 2012 cacheable_pages[page_idx] = 2013 qdf_mem_malloc(pages->page_size); 2014 if (!cacheable_pages[page_idx]) 2015 goto page_alloc_fail; 2016 } 2017 pages->dma_pages = NULL; 2018 } else { 2019 pages->dma_pages = qdf_mem_malloc( 2020 pages->num_pages * sizeof(struct qdf_mem_dma_page_t)); 2021 if (!pages->dma_pages) 2022 goto out_fail; 2023 2024 dma_pages = pages->dma_pages; 2025 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 2026 dma_pages->page_v_addr_start = 2027 qdf_mem_alloc_consistent(osdev, osdev->dev, 2028 pages->page_size, 2029 &dma_pages->page_p_addr); 2030 if (!dma_pages->page_v_addr_start) { 2031 qdf_print("dmaable page alloc fail pi %d", 2032 page_idx); 2033 goto page_alloc_fail; 2034 } 2035 dma_pages->page_v_addr_end = 2036 dma_pages->page_v_addr_start + pages->page_size; 2037 dma_pages++; 2038 } 2039 pages->cacheable_pages = NULL; 2040 } 2041 return; 2042 2043 page_alloc_fail: 2044 if (cacheable) { 2045 for (i = 0; i < page_idx; i++) 2046 qdf_mem_free(pages->cacheable_pages[i]); 2047 qdf_mem_free(pages->cacheable_pages); 2048 } else { 2049 dma_pages = pages->dma_pages; 2050 for (i = 0; i < page_idx; i++) { 2051 qdf_mem_free_consistent( 2052 osdev, osdev->dev, pages->page_size, 2053 dma_pages->page_v_addr_start, 2054 dma_pages->page_p_addr, memctxt); 2055 dma_pages++; 2056 } 2057 qdf_mem_free(pages->dma_pages); 2058 } 2059 2060 out_fail: 2061 pages->cacheable_pages = NULL; 2062 pages->dma_pages = NULL; 2063 pages->num_pages = 0; 2064 return; 2065 } 2066 qdf_export_symbol(qdf_mem_multi_pages_alloc); 2067 2068 /** 2069 * qdf_mem_multi_pages_free() - free large size of kernel memory 2070 * @osdev: OS device handle pointer 2071 * @pages: Multi page information storage 2072 * @memctxt: Memory context 2073 * @cacheable: Coherent memory or cacheable memory 2074 * 2075 * This function will free large size of memory over multiple pages. 2076 * 2077 * Return: None 2078 */ 2079 void qdf_mem_multi_pages_free(qdf_device_t osdev, 2080 struct qdf_mem_multi_page_t *pages, 2081 qdf_dma_context_t memctxt, bool cacheable) 2082 { 2083 unsigned int page_idx; 2084 struct qdf_mem_dma_page_t *dma_pages; 2085 2086 if (!pages->page_size) 2087 pages->page_size = qdf_page_size; 2088 2089 if (cacheable) { 2090 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) 2091 qdf_mem_free(pages->cacheable_pages[page_idx]); 2092 qdf_mem_free(pages->cacheable_pages); 2093 } else { 2094 dma_pages = pages->dma_pages; 2095 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 2096 qdf_mem_free_consistent( 2097 osdev, osdev->dev, pages->page_size, 2098 dma_pages->page_v_addr_start, 2099 dma_pages->page_p_addr, memctxt); 2100 dma_pages++; 2101 } 2102 qdf_mem_free(pages->dma_pages); 2103 } 2104 2105 pages->cacheable_pages = NULL; 2106 pages->dma_pages = NULL; 2107 pages->num_pages = 0; 2108 return; 2109 } 2110 qdf_export_symbol(qdf_mem_multi_pages_free); 2111 #endif 2112 2113 void qdf_mem_multi_pages_zero(struct qdf_mem_multi_page_t *pages, 2114 bool cacheable) 2115 { 2116 unsigned int page_idx; 2117 struct qdf_mem_dma_page_t *dma_pages; 2118 2119 if (!pages->page_size) 2120 pages->page_size = qdf_page_size; 2121 2122 if (cacheable) { 2123 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) 2124 qdf_mem_zero(pages->cacheable_pages[page_idx], 2125 pages->page_size); 2126 } else { 2127 dma_pages = pages->dma_pages; 2128 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 2129 qdf_mem_zero(dma_pages->page_v_addr_start, 2130 pages->page_size); 2131 dma_pages++; 2132 } 2133 } 2134 } 2135 2136 qdf_export_symbol(qdf_mem_multi_pages_zero); 2137 2138 void __qdf_mem_free(void *ptr) 2139 { 2140 if (!ptr) 2141 return; 2142 2143 if (qdf_might_be_prealloc(ptr)) { 2144 if (qdf_mem_prealloc_put(ptr)) 2145 return; 2146 } 2147 2148 qdf_mem_kmalloc_dec(ksize(ptr)); 2149 2150 kfree(ptr); 2151 } 2152 2153 qdf_export_symbol(__qdf_mem_free); 2154 2155 void *__qdf_mem_malloc(size_t size, const char *func, uint32_t line) 2156 { 2157 void *ptr; 2158 2159 if (!size || size > QDF_MEM_MAX_MALLOC) { 2160 qdf_nofl_err("Cannot malloc %zu bytes @ %s:%d", size, func, 2161 line); 2162 return NULL; 2163 } 2164 2165 ptr = qdf_mem_prealloc_get(size); 2166 if (ptr) 2167 return ptr; 2168 2169 ptr = kzalloc(size, qdf_mem_malloc_flags()); 2170 if (!ptr) 2171 return NULL; 2172 2173 qdf_mem_kmalloc_inc(ksize(ptr)); 2174 2175 return ptr; 2176 } 2177 2178 qdf_export_symbol(__qdf_mem_malloc); 2179 2180 #ifdef QCA_WIFI_MODULE_PARAMS_FROM_INI 2181 void __qdf_untracked_mem_free(void *ptr) 2182 { 2183 if (!ptr) 2184 return; 2185 2186 kfree(ptr); 2187 } 2188 2189 void *__qdf_untracked_mem_malloc(size_t size, const char *func, uint32_t line) 2190 { 2191 void *ptr; 2192 2193 if (!size || size > QDF_MEM_MAX_MALLOC) { 2194 qdf_nofl_err("Cannot malloc %zu bytes @ %s:%d", size, func, 2195 line); 2196 return NULL; 2197 } 2198 2199 ptr = kzalloc(size, qdf_mem_malloc_flags()); 2200 if (!ptr) 2201 return NULL; 2202 2203 return ptr; 2204 } 2205 #endif 2206 2207 void *qdf_aligned_malloc_fl(uint32_t *size, 2208 void **vaddr_unaligned, 2209 qdf_dma_addr_t *paddr_unaligned, 2210 qdf_dma_addr_t *paddr_aligned, 2211 uint32_t align, 2212 const char *func, uint32_t line) 2213 { 2214 void *vaddr_aligned; 2215 uint32_t align_alloc_size; 2216 2217 *vaddr_unaligned = qdf_mem_malloc_fl((qdf_size_t)*size, func, 2218 line); 2219 if (!*vaddr_unaligned) { 2220 qdf_warn("Failed to alloc %uB @ %s:%d", *size, func, line); 2221 return NULL; 2222 } 2223 2224 *paddr_unaligned = qdf_mem_virt_to_phys(*vaddr_unaligned); 2225 2226 /* Re-allocate additional bytes to align base address only if 2227 * above allocation returns unaligned address. Reason for 2228 * trying exact size allocation above is, OS tries to allocate 2229 * blocks of size power-of-2 pages and then free extra pages. 2230 * e.g., of a ring size of 1MB, the allocation below will 2231 * request 1MB plus 7 bytes for alignment, which will cause a 2232 * 2MB block allocation,and that is failing sometimes due to 2233 * memory fragmentation. 2234 */ 2235 if ((unsigned long)(*paddr_unaligned) & (align - 1)) { 2236 align_alloc_size = *size + align - 1; 2237 2238 qdf_mem_free(*vaddr_unaligned); 2239 *vaddr_unaligned = qdf_mem_malloc_fl( 2240 (qdf_size_t)align_alloc_size, func, line); 2241 if (!*vaddr_unaligned) { 2242 qdf_warn("Failed to alloc %uB @ %s:%d", 2243 align_alloc_size, func, line); 2244 return NULL; 2245 } 2246 2247 *paddr_unaligned = qdf_mem_virt_to_phys( 2248 *vaddr_unaligned); 2249 *size = align_alloc_size; 2250 } 2251 2252 *paddr_aligned = (qdf_dma_addr_t)qdf_align 2253 ((unsigned long)(*paddr_unaligned), align); 2254 2255 vaddr_aligned = (void *)((unsigned long)(*vaddr_unaligned) + 2256 ((unsigned long)(*paddr_aligned) - 2257 (unsigned long)(*paddr_unaligned))); 2258 2259 return vaddr_aligned; 2260 } 2261 2262 qdf_export_symbol(qdf_aligned_malloc_fl); 2263 2264 /** 2265 * qdf_mem_multi_page_link() - Make links for multi page elements 2266 * @osdev: OS device handle pointer 2267 * @pages: Multi page information storage 2268 * @elem_size: Single element size 2269 * @elem_count: elements count should be linked 2270 * @cacheable: Coherent memory or cacheable memory 2271 * 2272 * This function will make links for multi page allocated structure 2273 * 2274 * Return: 0 success 2275 */ 2276 int qdf_mem_multi_page_link(qdf_device_t osdev, 2277 struct qdf_mem_multi_page_t *pages, 2278 uint32_t elem_size, uint32_t elem_count, uint8_t cacheable) 2279 { 2280 uint16_t i, i_int; 2281 void *page_info; 2282 void **c_elem = NULL; 2283 uint32_t num_link = 0; 2284 2285 for (i = 0; i < pages->num_pages; i++) { 2286 if (cacheable) 2287 page_info = pages->cacheable_pages[i]; 2288 else 2289 page_info = pages->dma_pages[i].page_v_addr_start; 2290 2291 if (!page_info) 2292 return -ENOMEM; 2293 2294 c_elem = (void **)page_info; 2295 for (i_int = 0; i_int < pages->num_element_per_page; i_int++) { 2296 if (i_int == (pages->num_element_per_page - 1)) { 2297 if ((i + 1) == pages->num_pages) 2298 break; 2299 if (cacheable) 2300 *c_elem = pages-> 2301 cacheable_pages[i + 1]; 2302 else 2303 *c_elem = pages-> 2304 dma_pages[i + 1]. 2305 page_v_addr_start; 2306 num_link++; 2307 break; 2308 } else { 2309 *c_elem = 2310 (void *)(((char *)c_elem) + elem_size); 2311 } 2312 num_link++; 2313 c_elem = (void **)*c_elem; 2314 2315 /* Last link established exit */ 2316 if (num_link == (elem_count - 1)) 2317 break; 2318 } 2319 } 2320 2321 if (c_elem) 2322 *c_elem = NULL; 2323 2324 return 0; 2325 } 2326 qdf_export_symbol(qdf_mem_multi_page_link); 2327 2328 void qdf_mem_copy(void *dst_addr, const void *src_addr, uint32_t num_bytes) 2329 { 2330 /* special case where dst_addr or src_addr can be NULL */ 2331 if (!num_bytes) 2332 return; 2333 2334 QDF_BUG(dst_addr); 2335 QDF_BUG(src_addr); 2336 if (!dst_addr || !src_addr) 2337 return; 2338 2339 memcpy(dst_addr, src_addr, num_bytes); 2340 } 2341 qdf_export_symbol(qdf_mem_copy); 2342 2343 qdf_shared_mem_t *qdf_mem_shared_mem_alloc(qdf_device_t osdev, uint32_t size) 2344 { 2345 qdf_shared_mem_t *shared_mem; 2346 qdf_dma_addr_t dma_addr, paddr; 2347 int ret; 2348 2349 shared_mem = qdf_mem_malloc(sizeof(*shared_mem)); 2350 if (!shared_mem) 2351 return NULL; 2352 2353 shared_mem->vaddr = qdf_mem_alloc_consistent(osdev, osdev->dev, 2354 size, qdf_mem_get_dma_addr_ptr(osdev, 2355 &shared_mem->mem_info)); 2356 if (!shared_mem->vaddr) { 2357 qdf_err("Unable to allocate DMA memory for shared resource"); 2358 qdf_mem_free(shared_mem); 2359 return NULL; 2360 } 2361 2362 qdf_mem_set_dma_size(osdev, &shared_mem->mem_info, size); 2363 size = qdf_mem_get_dma_size(osdev, &shared_mem->mem_info); 2364 2365 qdf_mem_zero(shared_mem->vaddr, size); 2366 dma_addr = qdf_mem_get_dma_addr(osdev, &shared_mem->mem_info); 2367 paddr = qdf_mem_paddr_from_dmaaddr(osdev, dma_addr); 2368 2369 qdf_mem_set_dma_pa(osdev, &shared_mem->mem_info, paddr); 2370 ret = qdf_mem_dma_get_sgtable(osdev->dev, &shared_mem->sgtable, 2371 shared_mem->vaddr, dma_addr, size); 2372 if (ret) { 2373 qdf_err("Unable to get DMA sgtable"); 2374 qdf_mem_free_consistent(osdev, osdev->dev, 2375 shared_mem->mem_info.size, 2376 shared_mem->vaddr, 2377 dma_addr, 2378 qdf_get_dma_mem_context(shared_mem, 2379 memctx)); 2380 qdf_mem_free(shared_mem); 2381 return NULL; 2382 } 2383 2384 qdf_dma_get_sgtable_dma_addr(&shared_mem->sgtable); 2385 2386 return shared_mem; 2387 } 2388 2389 qdf_export_symbol(qdf_mem_shared_mem_alloc); 2390 2391 /** 2392 * qdf_mem_copy_toio() - copy memory 2393 * @dst_addr: Pointer to destination memory location (to copy to) 2394 * @src_addr: Pointer to source memory location (to copy from) 2395 * @num_bytes: Number of bytes to copy. 2396 * 2397 * Return: none 2398 */ 2399 void qdf_mem_copy_toio(void *dst_addr, const void *src_addr, uint32_t num_bytes) 2400 { 2401 if (0 == num_bytes) { 2402 /* special case where dst_addr or src_addr can be NULL */ 2403 return; 2404 } 2405 2406 if ((!dst_addr) || (!src_addr)) { 2407 QDF_TRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, 2408 "%s called with NULL parameter, source:%pK destination:%pK", 2409 __func__, src_addr, dst_addr); 2410 QDF_ASSERT(0); 2411 return; 2412 } 2413 memcpy_toio(dst_addr, src_addr, num_bytes); 2414 } 2415 2416 qdf_export_symbol(qdf_mem_copy_toio); 2417 2418 /** 2419 * qdf_mem_set_io() - set (fill) memory with a specified byte value. 2420 * @ptr: Pointer to memory that will be set 2421 * @value: Byte set in memory 2422 * @num_bytes: Number of bytes to be set 2423 * 2424 * Return: None 2425 */ 2426 void qdf_mem_set_io(void *ptr, uint32_t num_bytes, uint32_t value) 2427 { 2428 if (!ptr) { 2429 qdf_print("%s called with NULL parameter ptr", __func__); 2430 return; 2431 } 2432 memset_io(ptr, value, num_bytes); 2433 } 2434 2435 qdf_export_symbol(qdf_mem_set_io); 2436 2437 void qdf_mem_set(void *ptr, uint32_t num_bytes, uint32_t value) 2438 { 2439 QDF_BUG(ptr); 2440 if (!ptr) 2441 return; 2442 2443 memset(ptr, value, num_bytes); 2444 } 2445 qdf_export_symbol(qdf_mem_set); 2446 2447 void qdf_mem_move(void *dst_addr, const void *src_addr, uint32_t num_bytes) 2448 { 2449 /* special case where dst_addr or src_addr can be NULL */ 2450 if (!num_bytes) 2451 return; 2452 2453 QDF_BUG(dst_addr); 2454 QDF_BUG(src_addr); 2455 if (!dst_addr || !src_addr) 2456 return; 2457 2458 memmove(dst_addr, src_addr, num_bytes); 2459 } 2460 qdf_export_symbol(qdf_mem_move); 2461 2462 int qdf_mem_cmp(const void *left, const void *right, size_t size) 2463 { 2464 QDF_BUG(left); 2465 QDF_BUG(right); 2466 2467 return memcmp(left, right, size); 2468 } 2469 qdf_export_symbol(qdf_mem_cmp); 2470 2471 #if defined(A_SIMOS_DEVHOST) || defined(HIF_SDIO) || defined(HIF_USB) 2472 /** 2473 * qdf_mem_dma_alloc() - allocates memory for dma 2474 * @osdev: OS device handle 2475 * @dev: Pointer to device handle 2476 * @size: Size to be allocated 2477 * @phy_addr: Physical address 2478 * 2479 * Return: pointer of allocated memory or null if memory alloc fails 2480 */ 2481 static inline void *qdf_mem_dma_alloc(qdf_device_t osdev, void *dev, 2482 qdf_size_t size, 2483 qdf_dma_addr_t *phy_addr) 2484 { 2485 void *vaddr; 2486 2487 vaddr = qdf_mem_malloc(size); 2488 *phy_addr = ((uintptr_t) vaddr); 2489 /* using this type conversion to suppress "cast from pointer to integer 2490 * of different size" warning on some platforms 2491 */ 2492 BUILD_BUG_ON(sizeof(*phy_addr) < sizeof(vaddr)); 2493 return vaddr; 2494 } 2495 2496 #elif defined(CONFIG_WIFI_EMULATION_WIFI_3_0) && defined(BUILD_X86) && \ 2497 !defined(QCA_WIFI_QCN9000) 2498 2499 #define QCA8074_RAM_BASE 0x50000000 2500 #define QDF_MEM_ALLOC_X86_MAX_RETRIES 10 2501 void *qdf_mem_dma_alloc(qdf_device_t osdev, void *dev, qdf_size_t size, 2502 qdf_dma_addr_t *phy_addr) 2503 { 2504 void *vaddr = NULL; 2505 int i; 2506 2507 *phy_addr = 0; 2508 2509 for (i = 0; i < QDF_MEM_ALLOC_X86_MAX_RETRIES; i++) { 2510 vaddr = dma_alloc_coherent(dev, size, phy_addr, 2511 qdf_mem_malloc_flags()); 2512 2513 if (!vaddr) { 2514 qdf_err("%s failed , size: %zu!", __func__, size); 2515 return NULL; 2516 } 2517 2518 if (*phy_addr >= QCA8074_RAM_BASE) 2519 return vaddr; 2520 2521 dma_free_coherent(dev, size, vaddr, *phy_addr); 2522 } 2523 2524 return NULL; 2525 } 2526 2527 #else 2528 static inline void *qdf_mem_dma_alloc(qdf_device_t osdev, void *dev, 2529 qdf_size_t size, qdf_dma_addr_t *paddr) 2530 { 2531 return dma_alloc_coherent(dev, size, paddr, qdf_mem_malloc_flags()); 2532 } 2533 #endif 2534 2535 #if defined(A_SIMOS_DEVHOST) || defined(HIF_SDIO) || defined(HIF_USB) 2536 static inline void 2537 qdf_mem_dma_free(void *dev, qdf_size_t size, void *vaddr, qdf_dma_addr_t paddr) 2538 { 2539 qdf_mem_free(vaddr); 2540 } 2541 #else 2542 2543 static inline void 2544 qdf_mem_dma_free(void *dev, qdf_size_t size, void *vaddr, qdf_dma_addr_t paddr) 2545 { 2546 dma_free_coherent(dev, size, vaddr, paddr); 2547 } 2548 #endif 2549 2550 #ifdef MEMORY_DEBUG 2551 void *qdf_mem_alloc_consistent_debug(qdf_device_t osdev, void *dev, 2552 qdf_size_t size, qdf_dma_addr_t *paddr, 2553 const char *func, uint32_t line, 2554 void *caller) 2555 { 2556 QDF_STATUS status; 2557 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 2558 qdf_list_t *mem_list = qdf_mem_dma_list(current_domain); 2559 struct qdf_mem_header *header; 2560 void *vaddr; 2561 2562 if (is_initial_mem_debug_disabled) 2563 return __qdf_mem_alloc_consistent(osdev, dev, 2564 size, paddr, 2565 func, line); 2566 2567 if (!size || size > QDF_MEM_MAX_MALLOC) { 2568 qdf_err("Cannot malloc %zu bytes @ %s:%d", size, func, line); 2569 return NULL; 2570 } 2571 2572 vaddr = qdf_mem_dma_alloc(osdev, dev, size + QDF_DMA_MEM_DEBUG_SIZE, 2573 paddr); 2574 2575 if (!vaddr) { 2576 qdf_warn("Failed to malloc %zuB @ %s:%d", size, func, line); 2577 return NULL; 2578 } 2579 2580 header = qdf_mem_dma_get_header(vaddr, size); 2581 /* For DMA buffers we only add trailers, this function will init 2582 * the header structure at the tail 2583 * Prefix the header into DMA buffer causes SMMU faults, so 2584 * do not prefix header into the DMA buffers 2585 */ 2586 qdf_mem_header_init(header, size, func, line, caller); 2587 2588 qdf_spin_lock_irqsave(&qdf_mem_dma_list_lock); 2589 status = qdf_list_insert_front(mem_list, &header->node); 2590 qdf_spin_unlock_irqrestore(&qdf_mem_dma_list_lock); 2591 if (QDF_IS_STATUS_ERROR(status)) 2592 qdf_err("Failed to insert memory header; status %d", status); 2593 2594 qdf_mem_dma_inc(size); 2595 2596 return vaddr; 2597 } 2598 qdf_export_symbol(qdf_mem_alloc_consistent_debug); 2599 2600 void qdf_mem_free_consistent_debug(qdf_device_t osdev, void *dev, 2601 qdf_size_t size, void *vaddr, 2602 qdf_dma_addr_t paddr, 2603 qdf_dma_context_t memctx, 2604 const char *func, uint32_t line) 2605 { 2606 enum qdf_debug_domain domain = qdf_debug_domain_get(); 2607 struct qdf_mem_header *header; 2608 enum qdf_mem_validation_bitmap error_bitmap; 2609 2610 if (is_initial_mem_debug_disabled) { 2611 __qdf_mem_free_consistent( 2612 osdev, dev, 2613 size, vaddr, 2614 paddr, memctx); 2615 return; 2616 } 2617 2618 /* freeing a null pointer is valid */ 2619 if (qdf_unlikely(!vaddr)) 2620 return; 2621 2622 qdf_talloc_assert_no_children_fl(vaddr, func, line); 2623 2624 qdf_spin_lock_irqsave(&qdf_mem_dma_list_lock); 2625 /* For DMA buffers we only add trailers, this function will retrieve 2626 * the header structure at the tail 2627 * Prefix the header into DMA buffer causes SMMU faults, so 2628 * do not prefix header into the DMA buffers 2629 */ 2630 header = qdf_mem_dma_get_header(vaddr, size); 2631 error_bitmap = qdf_mem_header_validate(header, domain); 2632 if (!error_bitmap) { 2633 header->freed = true; 2634 qdf_list_remove_node(qdf_mem_dma_list(header->domain), 2635 &header->node); 2636 } 2637 qdf_spin_unlock_irqrestore(&qdf_mem_dma_list_lock); 2638 2639 qdf_mem_header_assert_valid(header, domain, error_bitmap, func, line); 2640 2641 qdf_mem_dma_dec(header->size); 2642 qdf_mem_dma_free(dev, size + QDF_DMA_MEM_DEBUG_SIZE, vaddr, paddr); 2643 } 2644 qdf_export_symbol(qdf_mem_free_consistent_debug); 2645 #endif /* MEMORY_DEBUG */ 2646 2647 void __qdf_mem_free_consistent(qdf_device_t osdev, void *dev, 2648 qdf_size_t size, void *vaddr, 2649 qdf_dma_addr_t paddr, qdf_dma_context_t memctx) 2650 { 2651 qdf_mem_dma_dec(size); 2652 qdf_mem_dma_free(dev, size, vaddr, paddr); 2653 } 2654 2655 qdf_export_symbol(__qdf_mem_free_consistent); 2656 2657 void *__qdf_mem_alloc_consistent(qdf_device_t osdev, void *dev, 2658 qdf_size_t size, qdf_dma_addr_t *paddr, 2659 const char *func, uint32_t line) 2660 { 2661 void *vaddr; 2662 2663 if (!size || size > QDF_MEM_MAX_MALLOC) { 2664 qdf_nofl_err("Cannot malloc %zu bytes @ %s:%d", 2665 size, func, line); 2666 return NULL; 2667 } 2668 2669 vaddr = qdf_mem_dma_alloc(osdev, dev, size, paddr); 2670 2671 if (vaddr) 2672 qdf_mem_dma_inc(size); 2673 2674 return vaddr; 2675 } 2676 2677 qdf_export_symbol(__qdf_mem_alloc_consistent); 2678 2679 void *qdf_aligned_mem_alloc_consistent_fl( 2680 qdf_device_t osdev, uint32_t *size, 2681 void **vaddr_unaligned, qdf_dma_addr_t *paddr_unaligned, 2682 qdf_dma_addr_t *paddr_aligned, uint32_t align, 2683 const char *func, uint32_t line) 2684 { 2685 void *vaddr_aligned; 2686 uint32_t align_alloc_size; 2687 2688 *vaddr_unaligned = qdf_mem_alloc_consistent( 2689 osdev, osdev->dev, (qdf_size_t)*size, paddr_unaligned); 2690 if (!*vaddr_unaligned) { 2691 qdf_warn("Failed to alloc %uB @ %s:%d", 2692 *size, func, line); 2693 return NULL; 2694 } 2695 2696 /* Re-allocate additional bytes to align base address only if 2697 * above allocation returns unaligned address. Reason for 2698 * trying exact size allocation above is, OS tries to allocate 2699 * blocks of size power-of-2 pages and then free extra pages. 2700 * e.g., of a ring size of 1MB, the allocation below will 2701 * request 1MB plus 7 bytes for alignment, which will cause a 2702 * 2MB block allocation,and that is failing sometimes due to 2703 * memory fragmentation. 2704 */ 2705 if ((unsigned long)(*paddr_unaligned) & (align - 1)) { 2706 align_alloc_size = *size + align - 1; 2707 2708 qdf_mem_free_consistent(osdev, osdev->dev, *size, 2709 *vaddr_unaligned, 2710 *paddr_unaligned, 0); 2711 2712 *vaddr_unaligned = qdf_mem_alloc_consistent( 2713 osdev, osdev->dev, align_alloc_size, 2714 paddr_unaligned); 2715 if (!*vaddr_unaligned) { 2716 qdf_warn("Failed to alloc %uB @ %s:%d", 2717 align_alloc_size, func, line); 2718 return NULL; 2719 } 2720 2721 *size = align_alloc_size; 2722 } 2723 2724 *paddr_aligned = (qdf_dma_addr_t)qdf_align( 2725 (unsigned long)(*paddr_unaligned), align); 2726 2727 vaddr_aligned = (void *)((unsigned long)(*vaddr_unaligned) + 2728 ((unsigned long)(*paddr_aligned) - 2729 (unsigned long)(*paddr_unaligned))); 2730 2731 return vaddr_aligned; 2732 } 2733 qdf_export_symbol(qdf_aligned_mem_alloc_consistent_fl); 2734 2735 /** 2736 * qdf_mem_dma_sync_single_for_device() - assign memory to device 2737 * @osdev: OS device handle 2738 * @bus_addr: dma address to give to the device 2739 * @size: Size of the memory block 2740 * @direction: direction data will be DMAed 2741 * 2742 * Assign memory to the remote device. 2743 * The cache lines are flushed to ram or invalidated as needed. 2744 * 2745 * Return: none 2746 */ 2747 void qdf_mem_dma_sync_single_for_device(qdf_device_t osdev, 2748 qdf_dma_addr_t bus_addr, 2749 qdf_size_t size, 2750 enum dma_data_direction direction) 2751 { 2752 dma_sync_single_for_device(osdev->dev, bus_addr, size, direction); 2753 } 2754 qdf_export_symbol(qdf_mem_dma_sync_single_for_device); 2755 2756 /** 2757 * qdf_mem_dma_sync_single_for_cpu() - assign memory to CPU 2758 * @osdev: OS device handle 2759 * @bus_addr: dma address to give to the cpu 2760 * @size: Size of the memory block 2761 * @direction: direction data will be DMAed 2762 * 2763 * Assign memory to the CPU. 2764 * 2765 * Return: none 2766 */ 2767 void qdf_mem_dma_sync_single_for_cpu(qdf_device_t osdev, 2768 qdf_dma_addr_t bus_addr, 2769 qdf_size_t size, 2770 enum dma_data_direction direction) 2771 { 2772 dma_sync_single_for_cpu(osdev->dev, bus_addr, size, direction); 2773 } 2774 qdf_export_symbol(qdf_mem_dma_sync_single_for_cpu); 2775 2776 void qdf_mem_init(void) 2777 { 2778 qdf_mem_debug_init(); 2779 qdf_net_buf_debug_init(); 2780 qdf_frag_debug_init(); 2781 qdf_mem_debugfs_init(); 2782 qdf_mem_debug_debugfs_init(); 2783 } 2784 qdf_export_symbol(qdf_mem_init); 2785 2786 void qdf_mem_exit(void) 2787 { 2788 qdf_mem_debug_debugfs_exit(); 2789 qdf_mem_debugfs_exit(); 2790 qdf_frag_debug_exit(); 2791 qdf_net_buf_debug_exit(); 2792 qdf_mem_debug_exit(); 2793 } 2794 qdf_export_symbol(qdf_mem_exit); 2795 2796 /** 2797 * qdf_ether_addr_copy() - copy an Ethernet address 2798 * 2799 * @dst_addr: A six-byte array Ethernet address destination 2800 * @src_addr: A six-byte array Ethernet address source 2801 * 2802 * Please note: dst & src must both be aligned to u16. 2803 * 2804 * Return: none 2805 */ 2806 void qdf_ether_addr_copy(void *dst_addr, const void *src_addr) 2807 { 2808 if ((!dst_addr) || (!src_addr)) { 2809 QDF_TRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, 2810 "%s called with NULL parameter, source:%pK destination:%pK", 2811 __func__, src_addr, dst_addr); 2812 QDF_ASSERT(0); 2813 return; 2814 } 2815 ether_addr_copy(dst_addr, src_addr); 2816 } 2817 qdf_export_symbol(qdf_ether_addr_copy); 2818 2819 int32_t qdf_dma_mem_stats_read(void) 2820 { 2821 return qdf_atomic_read(&qdf_mem_stat.dma); 2822 } 2823 2824 qdf_export_symbol(qdf_dma_mem_stats_read); 2825 2826 int32_t qdf_heap_mem_stats_read(void) 2827 { 2828 return qdf_atomic_read(&qdf_mem_stat.kmalloc); 2829 } 2830 2831 qdf_export_symbol(qdf_heap_mem_stats_read); 2832 2833 int32_t qdf_skb_mem_stats_read(void) 2834 { 2835 return qdf_atomic_read(&qdf_mem_stat.skb); 2836 } 2837 2838 qdf_export_symbol(qdf_skb_mem_stats_read); 2839 2840 int32_t qdf_skb_total_mem_stats_read(void) 2841 { 2842 return qdf_atomic_read(&qdf_mem_stat.skb_total); 2843 } 2844 2845 qdf_export_symbol(qdf_skb_total_mem_stats_read); 2846 2847 int32_t qdf_skb_max_mem_stats_read(void) 2848 { 2849 return qdf_mem_stat.skb_mem_max; 2850 } 2851 2852 qdf_export_symbol(qdf_skb_max_mem_stats_read); 2853 2854 int32_t qdf_dp_tx_skb_mem_stats_read(void) 2855 { 2856 return qdf_atomic_read(&qdf_mem_stat.dp_tx_skb); 2857 } 2858 2859 qdf_export_symbol(qdf_dp_tx_skb_mem_stats_read); 2860 2861 int32_t qdf_dp_rx_skb_mem_stats_read(void) 2862 { 2863 return qdf_atomic_read(&qdf_mem_stat.dp_rx_skb); 2864 } 2865 2866 qdf_export_symbol(qdf_dp_rx_skb_mem_stats_read); 2867 2868 int32_t qdf_mem_dp_tx_skb_cnt_read(void) 2869 { 2870 return qdf_atomic_read(&qdf_mem_stat.dp_tx_skb_count); 2871 } 2872 2873 qdf_export_symbol(qdf_mem_dp_tx_skb_cnt_read); 2874 2875 int32_t qdf_mem_dp_tx_skb_max_cnt_read(void) 2876 { 2877 return qdf_mem_stat.dp_tx_skb_count_max; 2878 } 2879 2880 qdf_export_symbol(qdf_mem_dp_tx_skb_max_cnt_read); 2881 2882 int32_t qdf_mem_dp_rx_skb_cnt_read(void) 2883 { 2884 return qdf_atomic_read(&qdf_mem_stat.dp_rx_skb_count); 2885 } 2886 2887 qdf_export_symbol(qdf_mem_dp_rx_skb_cnt_read); 2888 2889 int32_t qdf_mem_dp_rx_skb_max_cnt_read(void) 2890 { 2891 return qdf_mem_stat.dp_rx_skb_count_max; 2892 } 2893 2894 qdf_export_symbol(qdf_mem_dp_rx_skb_max_cnt_read); 2895 2896 int32_t qdf_dp_tx_skb_max_mem_stats_read(void) 2897 { 2898 return qdf_mem_stat.dp_tx_skb_mem_max; 2899 } 2900 2901 qdf_export_symbol(qdf_dp_tx_skb_max_mem_stats_read); 2902 2903 int32_t qdf_dp_rx_skb_max_mem_stats_read(void) 2904 { 2905 return qdf_mem_stat.dp_rx_skb_mem_max; 2906 } 2907 2908 qdf_export_symbol(qdf_dp_rx_skb_max_mem_stats_read); 2909 2910 int32_t qdf_mem_tx_desc_cnt_read(void) 2911 { 2912 return qdf_atomic_read(&qdf_mem_stat.tx_descs_outstanding); 2913 } 2914 2915 qdf_export_symbol(qdf_mem_tx_desc_cnt_read); 2916 2917 int32_t qdf_mem_tx_desc_max_read(void) 2918 { 2919 return qdf_mem_stat.tx_descs_max; 2920 } 2921 2922 qdf_export_symbol(qdf_mem_tx_desc_max_read); 2923 2924 void qdf_mem_tx_desc_cnt_update(qdf_atomic_t pending_tx_descs, 2925 int32_t tx_descs_max) 2926 { 2927 qdf_mem_stat.tx_descs_outstanding = pending_tx_descs; 2928 qdf_mem_stat.tx_descs_max = tx_descs_max; 2929 } 2930 2931 qdf_export_symbol(qdf_mem_tx_desc_cnt_update); 2932 2933 void qdf_mem_stats_init(void) 2934 { 2935 qdf_mem_stat.skb_mem_max = 0; 2936 qdf_mem_stat.dp_tx_skb_mem_max = 0; 2937 qdf_mem_stat.dp_rx_skb_mem_max = 0; 2938 qdf_mem_stat.dp_tx_skb_count_max = 0; 2939 qdf_mem_stat.dp_rx_skb_count_max = 0; 2940 qdf_mem_stat.tx_descs_max = 0; 2941 } 2942 2943 qdf_export_symbol(qdf_mem_stats_init); 2944 2945 void *__qdf_mem_valloc(size_t size, const char *func, uint32_t line) 2946 { 2947 void *ptr; 2948 2949 if (!size) { 2950 qdf_err("Valloc called with 0 bytes @ %s:%d", func, line); 2951 return NULL; 2952 } 2953 2954 ptr = vzalloc(size); 2955 2956 return ptr; 2957 } 2958 2959 qdf_export_symbol(__qdf_mem_valloc); 2960 2961 void __qdf_mem_vfree(void *ptr) 2962 { 2963 if (qdf_unlikely(!ptr)) 2964 return; 2965 2966 vfree(ptr); 2967 } 2968 2969 qdf_export_symbol(__qdf_mem_vfree); 2970 2971 #if IS_ENABLED(CONFIG_ARM_SMMU) && defined(ENABLE_SMMU_S1_TRANSLATION) 2972 int 2973 qdf_iommu_domain_get_attr(qdf_iommu_domain_t *domain, 2974 enum qdf_iommu_attr attr, void *data) 2975 { 2976 return __qdf_iommu_domain_get_attr(domain, attr, data); 2977 } 2978 2979 qdf_export_symbol(qdf_iommu_domain_get_attr); 2980 #endif 2981 2982 #ifdef ENHANCED_OS_ABSTRACTION 2983 void qdf_update_mem_map_table(qdf_device_t osdev, 2984 qdf_mem_info_t *mem_info, 2985 qdf_dma_addr_t dma_addr, 2986 uint32_t mem_size) 2987 { 2988 if (!mem_info) { 2989 qdf_nofl_err("%s: NULL mem_info", __func__); 2990 return; 2991 } 2992 2993 __qdf_update_mem_map_table(osdev, mem_info, dma_addr, mem_size); 2994 } 2995 2996 qdf_export_symbol(qdf_update_mem_map_table); 2997 2998 qdf_dma_addr_t qdf_mem_paddr_from_dmaaddr(qdf_device_t osdev, 2999 qdf_dma_addr_t dma_addr) 3000 { 3001 return __qdf_mem_paddr_from_dmaaddr(osdev, dma_addr); 3002 } 3003 3004 qdf_export_symbol(qdf_mem_paddr_from_dmaaddr); 3005 #endif 3006