1 /* 2 * Copyright (c) 2014-2019 The Linux Foundation. All rights reserved. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for 5 * any purpose with or without fee is hereby granted, provided that the 6 * above copyright notice and this permission notice appear in all 7 * copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 10 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 11 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 12 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 13 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 14 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 15 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 * PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /** 20 * DOC: qdf_mem 21 * This file provides OS dependent memory management APIs 22 */ 23 24 #include "qdf_debugfs.h" 25 #include "qdf_mem.h" 26 #include "qdf_nbuf.h" 27 #include "qdf_lock.h" 28 #include "qdf_mc_timer.h" 29 #include "qdf_module.h" 30 #include <qdf_trace.h> 31 #include "qdf_atomic.h" 32 #include "qdf_str.h" 33 #include "qdf_talloc.h" 34 #include <linux/debugfs.h> 35 #include <linux/seq_file.h> 36 #include <linux/string.h> 37 38 #if defined(CONFIG_CNSS) 39 #include <net/cnss.h> 40 #endif 41 42 #ifdef CONFIG_WCNSS_MEM_PRE_ALLOC 43 #include <net/cnss_prealloc.h> 44 #endif 45 46 #ifdef MEMORY_DEBUG 47 #include "qdf_debug_domain.h" 48 #include <qdf_list.h> 49 50 /* Preprocessor Definitions and Constants */ 51 #define QDF_MEM_MAX_MALLOC (4096 * 1024) /* 4 Mega Bytes */ 52 #define QDF_MEM_WARN_THRESHOLD 300 /* ms */ 53 #define QDF_DEBUG_STRING_SIZE 512 54 55 static qdf_list_t qdf_mem_domains[QDF_DEBUG_DOMAIN_COUNT]; 56 static qdf_spinlock_t qdf_mem_list_lock; 57 58 static qdf_list_t qdf_mem_dma_domains[QDF_DEBUG_DOMAIN_COUNT]; 59 static qdf_spinlock_t qdf_mem_dma_list_lock; 60 61 static inline qdf_list_t *qdf_mem_list_get(enum qdf_debug_domain domain) 62 { 63 return &qdf_mem_domains[domain]; 64 } 65 66 static inline qdf_list_t *qdf_mem_dma_list(enum qdf_debug_domain domain) 67 { 68 return &qdf_mem_dma_domains[domain]; 69 } 70 71 /** 72 * struct qdf_mem_header - memory object to dubug 73 * @node: node to the list 74 * @domain: the active memory domain at time of allocation 75 * @freed: flag set during free, used to detect double frees 76 * Use uint8_t so we can detect corruption 77 * @func: name of the function the allocation was made from 78 * @line: line number of the file the allocation was made from 79 * @size: size of the allocation in bytes 80 * @caller: Caller of the function for which memory is allocated 81 * @header: a known value, used to detect out-of-bounds access 82 * @time: timestamp at which allocation was made 83 */ 84 struct qdf_mem_header { 85 qdf_list_node_t node; 86 enum qdf_debug_domain domain; 87 uint8_t freed; 88 char func[QDF_MEM_FUNC_NAME_SIZE]; 89 uint32_t line; 90 uint32_t size; 91 void *caller; 92 uint64_t header; 93 uint64_t time; 94 }; 95 96 static uint64_t WLAN_MEM_HEADER = 0x6162636465666768; 97 static uint64_t WLAN_MEM_TRAILER = 0x8081828384858687; 98 99 static inline struct qdf_mem_header *qdf_mem_get_header(void *ptr) 100 { 101 return (struct qdf_mem_header *)ptr - 1; 102 } 103 104 static inline struct qdf_mem_header *qdf_mem_dma_get_header(void *ptr, 105 qdf_size_t size) 106 { 107 return (struct qdf_mem_header *) ((uint8_t *) ptr + size); 108 } 109 110 static inline uint64_t *qdf_mem_get_trailer(struct qdf_mem_header *header) 111 { 112 return (uint64_t *)((void *)(header + 1) + header->size); 113 } 114 115 static inline void *qdf_mem_get_ptr(struct qdf_mem_header *header) 116 { 117 return (void *)(header + 1); 118 } 119 120 /* number of bytes needed for the qdf memory debug information */ 121 #define QDF_MEM_DEBUG_SIZE \ 122 (sizeof(struct qdf_mem_header) + sizeof(WLAN_MEM_TRAILER)) 123 124 /* number of bytes needed for the qdf dma memory debug information */ 125 #define QDF_DMA_MEM_DEBUG_SIZE \ 126 (sizeof(struct qdf_mem_header)) 127 128 static void qdf_mem_trailer_init(struct qdf_mem_header *header) 129 { 130 QDF_BUG(header); 131 if (!header) 132 return; 133 *qdf_mem_get_trailer(header) = WLAN_MEM_TRAILER; 134 } 135 136 static void qdf_mem_header_init(struct qdf_mem_header *header, qdf_size_t size, 137 const char *func, uint32_t line, void *caller) 138 { 139 QDF_BUG(header); 140 if (!header) 141 return; 142 143 header->domain = qdf_debug_domain_get(); 144 header->freed = false; 145 146 qdf_str_lcopy(header->func, func, QDF_MEM_FUNC_NAME_SIZE); 147 148 header->line = line; 149 header->size = size; 150 header->caller = caller; 151 header->header = WLAN_MEM_HEADER; 152 header->time = qdf_get_log_timestamp(); 153 } 154 155 enum qdf_mem_validation_bitmap { 156 QDF_MEM_BAD_HEADER = 1 << 0, 157 QDF_MEM_BAD_TRAILER = 1 << 1, 158 QDF_MEM_BAD_SIZE = 1 << 2, 159 QDF_MEM_DOUBLE_FREE = 1 << 3, 160 QDF_MEM_BAD_FREED = 1 << 4, 161 QDF_MEM_BAD_NODE = 1 << 5, 162 QDF_MEM_BAD_DOMAIN = 1 << 6, 163 QDF_MEM_WRONG_DOMAIN = 1 << 7, 164 }; 165 166 static enum qdf_mem_validation_bitmap 167 qdf_mem_trailer_validate(struct qdf_mem_header *header) 168 { 169 enum qdf_mem_validation_bitmap error_bitmap = 0; 170 171 if (*qdf_mem_get_trailer(header) != WLAN_MEM_TRAILER) 172 error_bitmap |= QDF_MEM_BAD_TRAILER; 173 return error_bitmap; 174 } 175 176 static enum qdf_mem_validation_bitmap 177 qdf_mem_header_validate(struct qdf_mem_header *header, 178 enum qdf_debug_domain domain) 179 { 180 enum qdf_mem_validation_bitmap error_bitmap = 0; 181 182 if (header->header != WLAN_MEM_HEADER) 183 error_bitmap |= QDF_MEM_BAD_HEADER; 184 185 if (header->size > QDF_MEM_MAX_MALLOC) 186 error_bitmap |= QDF_MEM_BAD_SIZE; 187 188 if (header->freed == true) 189 error_bitmap |= QDF_MEM_DOUBLE_FREE; 190 else if (header->freed) 191 error_bitmap |= QDF_MEM_BAD_FREED; 192 193 if (!qdf_list_node_in_any_list(&header->node)) 194 error_bitmap |= QDF_MEM_BAD_NODE; 195 196 if (header->domain < QDF_DEBUG_DOMAIN_INIT || 197 header->domain >= QDF_DEBUG_DOMAIN_COUNT) 198 error_bitmap |= QDF_MEM_BAD_DOMAIN; 199 else if (header->domain != domain) 200 error_bitmap |= QDF_MEM_WRONG_DOMAIN; 201 202 return error_bitmap; 203 } 204 205 static void 206 qdf_mem_header_assert_valid(struct qdf_mem_header *header, 207 enum qdf_debug_domain current_domain, 208 enum qdf_mem_validation_bitmap error_bitmap, 209 const char *func, 210 uint32_t line) 211 { 212 if (!error_bitmap) 213 return; 214 215 if (error_bitmap & QDF_MEM_BAD_HEADER) 216 qdf_err("Corrupted memory header 0x%llx (expected 0x%llx)", 217 header->header, WLAN_MEM_HEADER); 218 219 if (error_bitmap & QDF_MEM_BAD_SIZE) 220 qdf_err("Corrupted memory size %u (expected < %d)", 221 header->size, QDF_MEM_MAX_MALLOC); 222 223 if (error_bitmap & QDF_MEM_BAD_TRAILER) 224 qdf_err("Corrupted memory trailer 0x%llx (expected 0x%llx)", 225 *qdf_mem_get_trailer(header), WLAN_MEM_TRAILER); 226 227 if (error_bitmap & QDF_MEM_DOUBLE_FREE) 228 qdf_err("Memory has previously been freed"); 229 230 if (error_bitmap & QDF_MEM_BAD_FREED) 231 qdf_err("Corrupted memory freed flag 0x%x", header->freed); 232 233 if (error_bitmap & QDF_MEM_BAD_NODE) 234 qdf_err("Corrupted memory header node or double free"); 235 236 if (error_bitmap & QDF_MEM_BAD_DOMAIN) 237 qdf_err("Corrupted memory domain 0x%x", header->domain); 238 239 if (error_bitmap & QDF_MEM_WRONG_DOMAIN) 240 qdf_err("Memory domain mismatch; allocated:%s(%d), current:%s(%d)", 241 qdf_debug_domain_name(header->domain), header->domain, 242 qdf_debug_domain_name(current_domain), current_domain); 243 244 QDF_DEBUG_PANIC("Fatal memory error detected @ %s:%d", func, line); 245 } 246 #endif /* MEMORY_DEBUG */ 247 248 u_int8_t prealloc_disabled = 1; 249 qdf_declare_param(prealloc_disabled, byte); 250 qdf_export_symbol(prealloc_disabled); 251 252 #if defined WLAN_DEBUGFS 253 254 /* Debugfs root directory for qdf_mem */ 255 static struct dentry *qdf_mem_debugfs_root; 256 257 /** 258 * struct __qdf_mem_stat - qdf memory statistics 259 * @kmalloc: total kmalloc allocations 260 * @dma: total dma allocations 261 * @skb: total skb allocations 262 */ 263 static struct __qdf_mem_stat { 264 qdf_atomic_t kmalloc; 265 qdf_atomic_t dma; 266 qdf_atomic_t skb; 267 } qdf_mem_stat; 268 269 void qdf_mem_kmalloc_inc(qdf_size_t size) 270 { 271 qdf_atomic_add(size, &qdf_mem_stat.kmalloc); 272 } 273 274 static void qdf_mem_dma_inc(qdf_size_t size) 275 { 276 qdf_atomic_add(size, &qdf_mem_stat.dma); 277 } 278 279 void qdf_mem_skb_inc(qdf_size_t size) 280 { 281 qdf_atomic_add(size, &qdf_mem_stat.skb); 282 } 283 284 void qdf_mem_kmalloc_dec(qdf_size_t size) 285 { 286 qdf_atomic_sub(size, &qdf_mem_stat.kmalloc); 287 } 288 289 static inline void qdf_mem_dma_dec(qdf_size_t size) 290 { 291 qdf_atomic_sub(size, &qdf_mem_stat.dma); 292 } 293 294 void qdf_mem_skb_dec(qdf_size_t size) 295 { 296 qdf_atomic_sub(size, &qdf_mem_stat.skb); 297 } 298 299 #ifdef MEMORY_DEBUG 300 static int qdf_err_printer(void *priv, const char *fmt, ...) 301 { 302 va_list args; 303 304 va_start(args, fmt); 305 QDF_VTRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, (char *)fmt, args); 306 va_end(args); 307 308 return 0; 309 } 310 311 static int seq_printf_printer(void *priv, const char *fmt, ...) 312 { 313 struct seq_file *file = priv; 314 va_list args; 315 316 va_start(args, fmt); 317 seq_vprintf(file, fmt, args); 318 seq_puts(file, "\n"); 319 va_end(args); 320 321 return 0; 322 } 323 324 /** 325 * struct __qdf_mem_info - memory statistics 326 * @func: the function which allocated memory 327 * @line: the line at which allocation happened 328 * @size: the size of allocation 329 * @caller: Address of the caller function 330 * @count: how many allocations of same type 331 * @time: timestamp at which allocation happened 332 */ 333 struct __qdf_mem_info { 334 char func[QDF_MEM_FUNC_NAME_SIZE]; 335 uint32_t line; 336 uint32_t size; 337 void *caller; 338 uint32_t count; 339 uint64_t time; 340 }; 341 342 /* 343 * The table depth defines the de-duplication proximity scope. 344 * A deeper table takes more time, so choose any optimum value. 345 */ 346 #define QDF_MEM_STAT_TABLE_SIZE 8 347 348 /** 349 * qdf_mem_domain_print_header() - memory domain header print logic 350 * @print: the print adapter function 351 * @print_priv: the private data to be consumed by @print 352 * 353 * Return: None 354 */ 355 static void qdf_mem_domain_print_header(qdf_abstract_print print, 356 void *print_priv) 357 { 358 print(print_priv, 359 "--------------------------------------------------------------"); 360 print(print_priv, 361 " count size total filename caller timestamp"); 362 print(print_priv, 363 "--------------------------------------------------------------"); 364 } 365 366 /** 367 * qdf_mem_meta_table_print() - memory metadata table print logic 368 * @table: the memory metadata table to print 369 * @print: the print adapter function 370 * @print_priv: the private data to be consumed by @print 371 * 372 * Return: None 373 */ 374 static void qdf_mem_meta_table_print(struct __qdf_mem_info *table, 375 qdf_abstract_print print, 376 void *print_priv) 377 { 378 int i; 379 char debug_str[QDF_DEBUG_STRING_SIZE]; 380 size_t len = 0; 381 char *debug_prefix = "WLAN_BUG_RCA: memory leak detected"; 382 383 len += qdf_scnprintf(debug_str, sizeof(debug_str) - len, 384 "%s", debug_prefix); 385 386 for (i = 0; i < QDF_MEM_STAT_TABLE_SIZE; i++) { 387 if (!table[i].count) 388 break; 389 390 print(print_priv, 391 "%6u x %5u = %7uB @ %s:%u %pS %llu", 392 table[i].count, 393 table[i].size, 394 table[i].count * table[i].size, 395 table[i].func, 396 table[i].line, table[i].caller, 397 table[i].time); 398 len += qdf_scnprintf(debug_str + len, 399 sizeof(debug_str) - len, 400 " @ %s:%u %pS", 401 table[i].func, 402 table[i].line, 403 table[i].caller); 404 } 405 print(print_priv, "%s", debug_str); 406 } 407 408 /** 409 * qdf_mem_meta_table_insert() - insert memory metadata into the given table 410 * @table: the memory metadata table to insert into 411 * @meta: the memory metadata to insert 412 * 413 * Return: true if the table is full after inserting, false otherwise 414 */ 415 static bool qdf_mem_meta_table_insert(struct __qdf_mem_info *table, 416 struct qdf_mem_header *meta) 417 { 418 int i; 419 420 for (i = 0; i < QDF_MEM_STAT_TABLE_SIZE; i++) { 421 if (!table[i].count) { 422 qdf_str_lcopy(table[i].func, meta->func, 423 QDF_MEM_FUNC_NAME_SIZE); 424 table[i].line = meta->line; 425 table[i].size = meta->size; 426 table[i].count = 1; 427 table[i].caller = meta->caller; 428 table[i].time = meta->time; 429 break; 430 } 431 432 if (qdf_str_eq(table[i].func, meta->func) && 433 table[i].line == meta->line && 434 table[i].size == meta->size && 435 table[i].caller == meta->caller) { 436 table[i].count++; 437 break; 438 } 439 } 440 441 /* return true if the table is now full */ 442 return i >= QDF_MEM_STAT_TABLE_SIZE - 1; 443 } 444 445 /** 446 * qdf_mem_domain_print() - output agnostic memory domain print logic 447 * @domain: the memory domain to print 448 * @print: the print adapter function 449 * @print_priv: the private data to be consumed by @print 450 * 451 * Return: None 452 */ 453 static void qdf_mem_domain_print(qdf_list_t *domain, 454 qdf_abstract_print print, 455 void *print_priv) 456 { 457 QDF_STATUS status; 458 struct __qdf_mem_info table[QDF_MEM_STAT_TABLE_SIZE]; 459 qdf_list_node_t *node; 460 461 qdf_mem_zero(table, sizeof(table)); 462 qdf_mem_domain_print_header(print, print_priv); 463 464 /* hold lock while inserting to avoid use-after free of the metadata */ 465 qdf_spin_lock(&qdf_mem_list_lock); 466 status = qdf_list_peek_front(domain, &node); 467 while (QDF_IS_STATUS_SUCCESS(status)) { 468 struct qdf_mem_header *meta = (struct qdf_mem_header *)node; 469 bool is_full = qdf_mem_meta_table_insert(table, meta); 470 471 qdf_spin_unlock(&qdf_mem_list_lock); 472 473 if (is_full) { 474 qdf_mem_meta_table_print(table, print, print_priv); 475 qdf_mem_zero(table, sizeof(table)); 476 } 477 478 qdf_spin_lock(&qdf_mem_list_lock); 479 status = qdf_list_peek_next(domain, node, &node); 480 } 481 qdf_spin_unlock(&qdf_mem_list_lock); 482 483 qdf_mem_meta_table_print(table, print, print_priv); 484 } 485 486 /** 487 * qdf_mem_seq_start() - sequential callback to start 488 * @seq: seq_file handle 489 * @pos: The start position of the sequence 490 * 491 * Return: iterator pointer, or NULL if iteration is complete 492 */ 493 static void *qdf_mem_seq_start(struct seq_file *seq, loff_t *pos) 494 { 495 enum qdf_debug_domain domain = *pos; 496 497 if (!qdf_debug_domain_valid(domain)) 498 return NULL; 499 500 /* just use the current position as our iterator */ 501 return pos; 502 } 503 504 /** 505 * qdf_mem_seq_next() - next sequential callback 506 * @seq: seq_file handle 507 * @v: the current iterator 508 * @pos: the current position 509 * 510 * Get the next node and release previous node. 511 * 512 * Return: iterator pointer, or NULL if iteration is complete 513 */ 514 static void *qdf_mem_seq_next(struct seq_file *seq, void *v, loff_t *pos) 515 { 516 ++*pos; 517 518 return qdf_mem_seq_start(seq, pos); 519 } 520 521 /** 522 * qdf_mem_seq_stop() - stop sequential callback 523 * @seq: seq_file handle 524 * @v: current iterator 525 * 526 * Return: None 527 */ 528 static void qdf_mem_seq_stop(struct seq_file *seq, void *v) { } 529 530 /** 531 * qdf_mem_seq_show() - print sequential callback 532 * @seq: seq_file handle 533 * @v: current iterator 534 * 535 * Return: 0 - success 536 */ 537 static int qdf_mem_seq_show(struct seq_file *seq, void *v) 538 { 539 enum qdf_debug_domain domain_id = *(enum qdf_debug_domain *)v; 540 541 seq_printf(seq, "\n%s Memory Domain (Id %d)\n", 542 qdf_debug_domain_name(domain_id), domain_id); 543 qdf_mem_domain_print(qdf_mem_list_get(domain_id), 544 seq_printf_printer, seq); 545 546 return 0; 547 } 548 549 /* sequential file operation table */ 550 static const struct seq_operations qdf_mem_seq_ops = { 551 .start = qdf_mem_seq_start, 552 .next = qdf_mem_seq_next, 553 .stop = qdf_mem_seq_stop, 554 .show = qdf_mem_seq_show, 555 }; 556 557 558 static int qdf_mem_debugfs_open(struct inode *inode, struct file *file) 559 { 560 return seq_open(file, &qdf_mem_seq_ops); 561 } 562 563 /* debugfs file operation table */ 564 static const struct file_operations fops_qdf_mem_debugfs = { 565 .owner = THIS_MODULE, 566 .open = qdf_mem_debugfs_open, 567 .read = seq_read, 568 .llseek = seq_lseek, 569 .release = seq_release, 570 }; 571 572 static QDF_STATUS qdf_mem_debug_debugfs_init(void) 573 { 574 if (!qdf_mem_debugfs_root) 575 return QDF_STATUS_E_FAILURE; 576 577 debugfs_create_file("list", 578 S_IRUSR, 579 qdf_mem_debugfs_root, 580 NULL, 581 &fops_qdf_mem_debugfs); 582 583 return QDF_STATUS_SUCCESS; 584 } 585 586 static QDF_STATUS qdf_mem_debug_debugfs_exit(void) 587 { 588 return QDF_STATUS_SUCCESS; 589 } 590 591 #else /* MEMORY_DEBUG */ 592 593 static QDF_STATUS qdf_mem_debug_debugfs_init(void) 594 { 595 return QDF_STATUS_E_NOSUPPORT; 596 } 597 598 static QDF_STATUS qdf_mem_debug_debugfs_exit(void) 599 { 600 return QDF_STATUS_E_NOSUPPORT; 601 } 602 603 #endif /* MEMORY_DEBUG */ 604 605 606 static void qdf_mem_debugfs_exit(void) 607 { 608 debugfs_remove_recursive(qdf_mem_debugfs_root); 609 qdf_mem_debugfs_root = NULL; 610 } 611 612 static QDF_STATUS qdf_mem_debugfs_init(void) 613 { 614 struct dentry *qdf_debugfs_root = qdf_debugfs_get_root(); 615 616 if (!qdf_debugfs_root) 617 return QDF_STATUS_E_FAILURE; 618 619 qdf_mem_debugfs_root = debugfs_create_dir("mem", qdf_debugfs_root); 620 621 if (!qdf_mem_debugfs_root) 622 return QDF_STATUS_E_FAILURE; 623 624 625 debugfs_create_atomic_t("kmalloc", 626 S_IRUSR, 627 qdf_mem_debugfs_root, 628 &qdf_mem_stat.kmalloc); 629 630 debugfs_create_atomic_t("dma", 631 S_IRUSR, 632 qdf_mem_debugfs_root, 633 &qdf_mem_stat.dma); 634 635 debugfs_create_atomic_t("skb", 636 S_IRUSR, 637 qdf_mem_debugfs_root, 638 &qdf_mem_stat.skb); 639 640 return QDF_STATUS_SUCCESS; 641 } 642 643 #else /* WLAN_DEBUGFS */ 644 645 static inline void qdf_mem_dma_inc(qdf_size_t size) {} 646 static inline void qdf_mem_dma_dec(qdf_size_t size) {} 647 648 static QDF_STATUS qdf_mem_debugfs_init(void) 649 { 650 return QDF_STATUS_E_NOSUPPORT; 651 } 652 static void qdf_mem_debugfs_exit(void) {} 653 654 655 static QDF_STATUS qdf_mem_debug_debugfs_init(void) 656 { 657 return QDF_STATUS_E_NOSUPPORT; 658 } 659 660 static QDF_STATUS qdf_mem_debug_debugfs_exit(void) 661 { 662 return QDF_STATUS_E_NOSUPPORT; 663 } 664 665 #endif /* WLAN_DEBUGFS */ 666 667 /** 668 * __qdf_mempool_init() - Create and initialize memory pool 669 * 670 * @osdev: platform device object 671 * @pool_addr: address of the pool created 672 * @elem_cnt: no. of elements in pool 673 * @elem_size: size of each pool element in bytes 674 * @flags: flags 675 * 676 * return: Handle to memory pool or NULL if allocation failed 677 */ 678 int __qdf_mempool_init(qdf_device_t osdev, __qdf_mempool_t *pool_addr, 679 int elem_cnt, size_t elem_size, u_int32_t flags) 680 { 681 __qdf_mempool_ctxt_t *new_pool = NULL; 682 u_int32_t align = L1_CACHE_BYTES; 683 unsigned long aligned_pool_mem; 684 int pool_id; 685 int i; 686 687 if (prealloc_disabled) { 688 /* TBD: We can maintain a list of pools in qdf_device_t 689 * to help debugging 690 * when pre-allocation is not enabled 691 */ 692 new_pool = (__qdf_mempool_ctxt_t *) 693 kmalloc(sizeof(__qdf_mempool_ctxt_t), GFP_KERNEL); 694 if (new_pool == NULL) 695 return QDF_STATUS_E_NOMEM; 696 697 memset(new_pool, 0, sizeof(*new_pool)); 698 /* TBD: define flags for zeroing buffers etc */ 699 new_pool->flags = flags; 700 new_pool->elem_size = elem_size; 701 new_pool->max_elem = elem_cnt; 702 *pool_addr = new_pool; 703 return 0; 704 } 705 706 for (pool_id = 0; pool_id < MAX_MEM_POOLS; pool_id++) { 707 if (osdev->mem_pool[pool_id] == NULL) 708 break; 709 } 710 711 if (pool_id == MAX_MEM_POOLS) 712 return -ENOMEM; 713 714 new_pool = osdev->mem_pool[pool_id] = (__qdf_mempool_ctxt_t *) 715 kmalloc(sizeof(__qdf_mempool_ctxt_t), GFP_KERNEL); 716 if (new_pool == NULL) 717 return -ENOMEM; 718 719 memset(new_pool, 0, sizeof(*new_pool)); 720 /* TBD: define flags for zeroing buffers etc */ 721 new_pool->flags = flags; 722 new_pool->pool_id = pool_id; 723 724 /* Round up the element size to cacheline */ 725 new_pool->elem_size = roundup(elem_size, L1_CACHE_BYTES); 726 new_pool->mem_size = elem_cnt * new_pool->elem_size + 727 ((align)?(align - 1):0); 728 729 new_pool->pool_mem = kzalloc(new_pool->mem_size, GFP_KERNEL); 730 if (new_pool->pool_mem == NULL) { 731 /* TBD: Check if we need get_free_pages above */ 732 kfree(new_pool); 733 osdev->mem_pool[pool_id] = NULL; 734 return -ENOMEM; 735 } 736 737 spin_lock_init(&new_pool->lock); 738 739 /* Initialize free list */ 740 aligned_pool_mem = (unsigned long)(new_pool->pool_mem) + 741 ((align) ? (unsigned long)(new_pool->pool_mem)%align:0); 742 STAILQ_INIT(&new_pool->free_list); 743 744 for (i = 0; i < elem_cnt; i++) 745 STAILQ_INSERT_TAIL(&(new_pool->free_list), 746 (mempool_elem_t *)(aligned_pool_mem + 747 (new_pool->elem_size * i)), mempool_entry); 748 749 750 new_pool->free_cnt = elem_cnt; 751 *pool_addr = new_pool; 752 return 0; 753 } 754 qdf_export_symbol(__qdf_mempool_init); 755 756 /** 757 * __qdf_mempool_destroy() - Destroy memory pool 758 * @osdev: platform device object 759 * @Handle: to memory pool 760 * 761 * Returns: none 762 */ 763 void __qdf_mempool_destroy(qdf_device_t osdev, __qdf_mempool_t pool) 764 { 765 int pool_id = 0; 766 767 if (!pool) 768 return; 769 770 if (prealloc_disabled) { 771 kfree(pool); 772 return; 773 } 774 775 pool_id = pool->pool_id; 776 777 /* TBD: Check if free count matches elem_cnt if debug is enabled */ 778 kfree(pool->pool_mem); 779 kfree(pool); 780 osdev->mem_pool[pool_id] = NULL; 781 } 782 qdf_export_symbol(__qdf_mempool_destroy); 783 784 /** 785 * __qdf_mempool_alloc() - Allocate an element memory pool 786 * 787 * @osdev: platform device object 788 * @Handle: to memory pool 789 * 790 * Return: Pointer to the allocated element or NULL if the pool is empty 791 */ 792 void *__qdf_mempool_alloc(qdf_device_t osdev, __qdf_mempool_t pool) 793 { 794 void *buf = NULL; 795 796 if (!pool) 797 return NULL; 798 799 if (prealloc_disabled) 800 return qdf_mem_malloc(pool->elem_size); 801 802 spin_lock_bh(&pool->lock); 803 804 buf = STAILQ_FIRST(&pool->free_list); 805 if (buf != NULL) { 806 STAILQ_REMOVE_HEAD(&pool->free_list, mempool_entry); 807 pool->free_cnt--; 808 } 809 810 /* TBD: Update free count if debug is enabled */ 811 spin_unlock_bh(&pool->lock); 812 813 return buf; 814 } 815 qdf_export_symbol(__qdf_mempool_alloc); 816 817 /** 818 * __qdf_mempool_free() - Free a memory pool element 819 * @osdev: Platform device object 820 * @pool: Handle to memory pool 821 * @buf: Element to be freed 822 * 823 * Returns: none 824 */ 825 void __qdf_mempool_free(qdf_device_t osdev, __qdf_mempool_t pool, void *buf) 826 { 827 if (!pool) 828 return; 829 830 831 if (prealloc_disabled) 832 return qdf_mem_free(buf); 833 834 spin_lock_bh(&pool->lock); 835 pool->free_cnt++; 836 837 STAILQ_INSERT_TAIL 838 (&pool->free_list, (mempool_elem_t *)buf, mempool_entry); 839 spin_unlock_bh(&pool->lock); 840 } 841 qdf_export_symbol(__qdf_mempool_free); 842 843 /** 844 * qdf_mem_alloc_outline() - allocation QDF memory 845 * @osdev: platform device object 846 * @size: Number of bytes of memory to allocate. 847 * 848 * This function will dynamicallly allocate the specified number of bytes of 849 * memory. 850 * 851 * Return: 852 * Upon successful allocate, returns a non-NULL pointer to the allocated 853 * memory. If this function is unable to allocate the amount of memory 854 * specified (for any reason) it returns NULL. 855 */ 856 void * 857 qdf_mem_alloc_outline(qdf_device_t osdev, size_t size) 858 { 859 return qdf_mem_malloc(size); 860 } 861 qdf_export_symbol(qdf_mem_alloc_outline); 862 863 /** 864 * qdf_mem_free_outline() - QDF memory free API 865 * @ptr: Pointer to the starting address of the memory to be free'd. 866 * 867 * This function will free the memory pointed to by 'ptr'. It also checks 868 * is memory is corrupted or getting double freed and panic. 869 * 870 * Return: none 871 */ 872 void 873 qdf_mem_free_outline(void *buf) 874 { 875 qdf_mem_free(buf); 876 } 877 qdf_export_symbol(qdf_mem_free_outline); 878 879 /** 880 * qdf_mem_zero_outline() - zero out memory 881 * @buf: pointer to memory that will be set to zero 882 * @size: number of bytes zero 883 * 884 * This function sets the memory location to all zeros, essentially clearing 885 * the memory. 886 * 887 * Return: none 888 */ 889 void 890 qdf_mem_zero_outline(void *buf, qdf_size_t size) 891 { 892 qdf_mem_zero(buf, size); 893 } 894 qdf_export_symbol(qdf_mem_zero_outline); 895 896 #ifdef CONFIG_WCNSS_MEM_PRE_ALLOC 897 /** 898 * qdf_mem_prealloc_get() - conditionally pre-allocate memory 899 * @size: the number of bytes to allocate 900 * 901 * If size if greater than WCNSS_PRE_ALLOC_GET_THRESHOLD, this function returns 902 * a chunk of pre-allocated memory. If size if less than or equal to 903 * WCNSS_PRE_ALLOC_GET_THRESHOLD, or an error occurs, NULL is returned instead. 904 * 905 * Return: NULL on failure, non-NULL on success 906 */ 907 static void *qdf_mem_prealloc_get(size_t size) 908 { 909 void *ptr; 910 911 if (size <= WCNSS_PRE_ALLOC_GET_THRESHOLD) 912 return NULL; 913 914 ptr = wcnss_prealloc_get(size); 915 if (!ptr) 916 return NULL; 917 918 memset(ptr, 0, size); 919 920 return ptr; 921 } 922 923 static inline bool qdf_mem_prealloc_put(void *ptr) 924 { 925 return wcnss_prealloc_put(ptr); 926 } 927 #else 928 static inline void *qdf_mem_prealloc_get(size_t size) 929 { 930 return NULL; 931 } 932 933 static inline bool qdf_mem_prealloc_put(void *ptr) 934 { 935 return false; 936 } 937 #endif /* CONFIG_WCNSS_MEM_PRE_ALLOC */ 938 939 static int qdf_mem_malloc_flags(void) 940 { 941 if (in_interrupt() || irqs_disabled() || in_atomic()) 942 return GFP_ATOMIC; 943 944 return GFP_KERNEL; 945 } 946 947 /* External Function implementation */ 948 #ifdef MEMORY_DEBUG 949 950 /** 951 * qdf_mem_debug_init() - initialize qdf memory debug functionality 952 * 953 * Return: none 954 */ 955 static void qdf_mem_debug_init(void) 956 { 957 int i; 958 959 /* Initalizing the list with maximum size of 60000 */ 960 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 961 qdf_list_create(&qdf_mem_domains[i], 60000); 962 qdf_spinlock_create(&qdf_mem_list_lock); 963 964 /* dma */ 965 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 966 qdf_list_create(&qdf_mem_dma_domains[i], 0); 967 qdf_spinlock_create(&qdf_mem_dma_list_lock); 968 } 969 970 static uint32_t 971 qdf_mem_domain_check_for_leaks(enum qdf_debug_domain domain, 972 qdf_list_t *mem_list) 973 { 974 if (qdf_list_empty(mem_list)) 975 return 0; 976 977 qdf_err("Memory leaks detected in %s domain!", 978 qdf_debug_domain_name(domain)); 979 qdf_mem_domain_print(mem_list, qdf_err_printer, NULL); 980 981 return mem_list->count; 982 } 983 984 static void qdf_mem_domain_set_check_for_leaks(qdf_list_t *domains) 985 { 986 uint32_t leak_count = 0; 987 int i; 988 989 /* detect and print leaks */ 990 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 991 leak_count += qdf_mem_domain_check_for_leaks(i, domains + i); 992 993 if (leak_count) 994 panic("%u fatal memory leaks detected!", leak_count); 995 } 996 997 /** 998 * qdf_mem_debug_exit() - exit qdf memory debug functionality 999 * 1000 * Return: none 1001 */ 1002 static void qdf_mem_debug_exit(void) 1003 { 1004 int i; 1005 1006 /* mem */ 1007 qdf_mem_domain_set_check_for_leaks(qdf_mem_domains); 1008 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 1009 qdf_list_destroy(qdf_mem_list_get(i)); 1010 1011 qdf_spinlock_destroy(&qdf_mem_list_lock); 1012 1013 /* dma */ 1014 qdf_mem_domain_set_check_for_leaks(qdf_mem_dma_domains); 1015 for (i = 0; i < QDF_DEBUG_DOMAIN_COUNT; ++i) 1016 qdf_list_destroy(&qdf_mem_dma_domains[i]); 1017 qdf_spinlock_destroy(&qdf_mem_dma_list_lock); 1018 } 1019 1020 void *qdf_mem_malloc_debug(size_t size, const char *func, uint32_t line, 1021 void *caller, uint32_t flag) 1022 { 1023 QDF_STATUS status; 1024 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1025 qdf_list_t *mem_list = qdf_mem_list_get(current_domain); 1026 struct qdf_mem_header *header; 1027 void *ptr; 1028 unsigned long start, duration; 1029 1030 if (!size || size > QDF_MEM_MAX_MALLOC) { 1031 qdf_err("Cannot malloc %zu bytes @ %s:%d", size, func, line); 1032 return NULL; 1033 } 1034 1035 ptr = qdf_mem_prealloc_get(size); 1036 if (ptr) 1037 return ptr; 1038 1039 if (!flag) 1040 flag = qdf_mem_malloc_flags(); 1041 1042 start = qdf_mc_timer_get_system_time(); 1043 header = kzalloc(size + QDF_MEM_DEBUG_SIZE, flag); 1044 duration = qdf_mc_timer_get_system_time() - start; 1045 1046 if (duration > QDF_MEM_WARN_THRESHOLD) 1047 qdf_warn("Malloc slept; %lums, %zuB @ %s:%d", 1048 duration, size, func, line); 1049 1050 if (!header) { 1051 qdf_warn("Failed to malloc %zuB @ %s:%d", size, func, line); 1052 return NULL; 1053 } 1054 1055 qdf_mem_header_init(header, size, func, line, caller); 1056 qdf_mem_trailer_init(header); 1057 ptr = qdf_mem_get_ptr(header); 1058 1059 qdf_spin_lock_irqsave(&qdf_mem_list_lock); 1060 status = qdf_list_insert_front(mem_list, &header->node); 1061 qdf_spin_unlock_irqrestore(&qdf_mem_list_lock); 1062 if (QDF_IS_STATUS_ERROR(status)) 1063 qdf_err("Failed to insert memory header; status %d", status); 1064 1065 qdf_mem_kmalloc_inc(size); 1066 1067 return ptr; 1068 } 1069 qdf_export_symbol(qdf_mem_malloc_debug); 1070 1071 void qdf_mem_free_debug(void *ptr, const char *func, uint32_t line) 1072 { 1073 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1074 struct qdf_mem_header *header; 1075 enum qdf_mem_validation_bitmap error_bitmap; 1076 1077 /* freeing a null pointer is valid */ 1078 if (qdf_unlikely(!ptr)) 1079 return; 1080 1081 if (qdf_mem_prealloc_put(ptr)) 1082 return; 1083 1084 if (qdf_unlikely((qdf_size_t)ptr <= sizeof(*header))) 1085 panic("Failed to free invalid memory location %pK", ptr); 1086 1087 qdf_talloc_assert_no_children_fl(ptr, func, line); 1088 1089 qdf_spin_lock_irqsave(&qdf_mem_list_lock); 1090 header = qdf_mem_get_header(ptr); 1091 error_bitmap = qdf_mem_header_validate(header, current_domain); 1092 error_bitmap |= qdf_mem_trailer_validate(header); 1093 1094 if (!error_bitmap) { 1095 header->freed = true; 1096 qdf_list_remove_node(qdf_mem_list_get(header->domain), 1097 &header->node); 1098 } 1099 qdf_spin_unlock_irqrestore(&qdf_mem_list_lock); 1100 1101 qdf_mem_header_assert_valid(header, current_domain, error_bitmap, 1102 func, line); 1103 1104 qdf_mem_kmalloc_dec(header->size); 1105 kfree(header); 1106 } 1107 qdf_export_symbol(qdf_mem_free_debug); 1108 1109 void qdf_mem_check_for_leaks(void) 1110 { 1111 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1112 qdf_list_t *mem_list = qdf_mem_list_get(current_domain); 1113 qdf_list_t *dma_list = qdf_mem_dma_list(current_domain); 1114 uint32_t leaks_count = 0; 1115 1116 leaks_count += qdf_mem_domain_check_for_leaks(current_domain, mem_list); 1117 leaks_count += qdf_mem_domain_check_for_leaks(current_domain, dma_list); 1118 1119 if (leaks_count) 1120 panic("%u fatal memory leaks detected!", leaks_count); 1121 } 1122 1123 #else 1124 static void qdf_mem_debug_init(void) {} 1125 1126 static void qdf_mem_debug_exit(void) {} 1127 1128 void *qdf_mem_malloc_fl(size_t size, const char *func, uint32_t line) 1129 { 1130 void *ptr; 1131 1132 ptr = qdf_mem_prealloc_get(size); 1133 if (ptr) 1134 return ptr; 1135 1136 ptr = kzalloc(size, qdf_mem_malloc_flags()); 1137 if (!ptr) { 1138 qdf_nofl_err("Failed to malloc %zuB @ %s:%d", 1139 size, func, line); 1140 return NULL; 1141 } 1142 1143 qdf_mem_kmalloc_inc(ksize(ptr)); 1144 1145 return ptr; 1146 } 1147 qdf_export_symbol(qdf_mem_malloc_fl); 1148 1149 void *qdf_mem_malloc_atomic_fl(size_t size, const char *func, uint32_t line) 1150 { 1151 void *ptr; 1152 1153 ptr = qdf_mem_prealloc_get(size); 1154 if (ptr) 1155 return ptr; 1156 1157 ptr = kzalloc(size, GFP_ATOMIC); 1158 if (!ptr) { 1159 qdf_nofl_warn("Failed to malloc %zuB @ %s:%d", 1160 size, func, line); 1161 return NULL; 1162 } 1163 1164 qdf_mem_kmalloc_inc(ksize(ptr)); 1165 1166 return ptr; 1167 } 1168 qdf_export_symbol(qdf_mem_malloc_atomic_fl); 1169 1170 /** 1171 * qdf_mem_free() - free QDF memory 1172 * @ptr: Pointer to the starting address of the memory to be free'd. 1173 * 1174 * This function will free the memory pointed to by 'ptr'. 1175 * 1176 * Return: None 1177 */ 1178 void qdf_mem_free(void *ptr) 1179 { 1180 if (!ptr) 1181 return; 1182 1183 if (qdf_mem_prealloc_put(ptr)) 1184 return; 1185 1186 qdf_mem_kmalloc_dec(ksize(ptr)); 1187 1188 kfree(ptr); 1189 } 1190 1191 qdf_export_symbol(qdf_mem_free); 1192 #endif 1193 1194 void *qdf_aligned_malloc_fl(qdf_size_t size, uint32_t ring_base_align, 1195 void **vaddr_unaligned, 1196 const char *func, uint32_t line) 1197 { 1198 void *vaddr_aligned; 1199 1200 *vaddr_unaligned = qdf_mem_malloc_fl(size, func, line); 1201 if (!*vaddr_unaligned) { 1202 qdf_warn("Failed to alloc %zuB @ %s:%d", size, func, line); 1203 return NULL; 1204 } 1205 1206 if ((unsigned long)(*vaddr_unaligned) % ring_base_align) { 1207 qdf_mem_free(*vaddr_unaligned); 1208 *vaddr_unaligned = qdf_mem_malloc_fl(size + ring_base_align - 1, 1209 func, line); 1210 if (!*vaddr_unaligned) { 1211 qdf_warn("Failed to alloc %zuB @ %s:%d", 1212 size, func, line); 1213 return NULL; 1214 } 1215 } 1216 1217 vaddr_aligned = (*vaddr_unaligned) + 1218 ((unsigned long)(*vaddr_unaligned) % ring_base_align); 1219 1220 return vaddr_aligned; 1221 } 1222 qdf_export_symbol(qdf_aligned_malloc_fl); 1223 1224 /** 1225 * qdf_mem_multi_pages_alloc() - allocate large size of kernel memory 1226 * @osdev: OS device handle pointer 1227 * @pages: Multi page information storage 1228 * @element_size: Each element size 1229 * @element_num: Total number of elements should be allocated 1230 * @memctxt: Memory context 1231 * @cacheable: Coherent memory or cacheable memory 1232 * 1233 * This function will allocate large size of memory over multiple pages. 1234 * Large size of contiguous memory allocation will fail frequently, then 1235 * instead of allocate large memory by one shot, allocate through multiple, non 1236 * contiguous memory and combine pages when actual usage 1237 * 1238 * Return: None 1239 */ 1240 void qdf_mem_multi_pages_alloc(qdf_device_t osdev, 1241 struct qdf_mem_multi_page_t *pages, 1242 size_t element_size, uint16_t element_num, 1243 qdf_dma_context_t memctxt, bool cacheable) 1244 { 1245 uint16_t page_idx; 1246 struct qdf_mem_dma_page_t *dma_pages; 1247 void **cacheable_pages = NULL; 1248 uint16_t i; 1249 1250 pages->num_element_per_page = PAGE_SIZE / element_size; 1251 if (!pages->num_element_per_page) { 1252 qdf_print("Invalid page %d or element size %d", 1253 (int)PAGE_SIZE, (int)element_size); 1254 goto out_fail; 1255 } 1256 1257 pages->num_pages = element_num / pages->num_element_per_page; 1258 if (element_num % pages->num_element_per_page) 1259 pages->num_pages++; 1260 1261 if (cacheable) { 1262 /* Pages information storage */ 1263 pages->cacheable_pages = qdf_mem_malloc( 1264 pages->num_pages * sizeof(pages->cacheable_pages)); 1265 if (!pages->cacheable_pages) 1266 goto out_fail; 1267 1268 cacheable_pages = pages->cacheable_pages; 1269 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 1270 cacheable_pages[page_idx] = qdf_mem_malloc(PAGE_SIZE); 1271 if (!cacheable_pages[page_idx]) 1272 goto page_alloc_fail; 1273 } 1274 pages->dma_pages = NULL; 1275 } else { 1276 pages->dma_pages = qdf_mem_malloc( 1277 pages->num_pages * sizeof(struct qdf_mem_dma_page_t)); 1278 if (!pages->dma_pages) 1279 goto out_fail; 1280 1281 dma_pages = pages->dma_pages; 1282 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 1283 dma_pages->page_v_addr_start = 1284 qdf_mem_alloc_consistent(osdev, osdev->dev, 1285 PAGE_SIZE, 1286 &dma_pages->page_p_addr); 1287 if (!dma_pages->page_v_addr_start) { 1288 qdf_print("dmaable page alloc fail pi %d", 1289 page_idx); 1290 goto page_alloc_fail; 1291 } 1292 dma_pages->page_v_addr_end = 1293 dma_pages->page_v_addr_start + PAGE_SIZE; 1294 dma_pages++; 1295 } 1296 pages->cacheable_pages = NULL; 1297 } 1298 return; 1299 1300 page_alloc_fail: 1301 if (cacheable) { 1302 for (i = 0; i < page_idx; i++) 1303 qdf_mem_free(pages->cacheable_pages[i]); 1304 qdf_mem_free(pages->cacheable_pages); 1305 } else { 1306 dma_pages = pages->dma_pages; 1307 for (i = 0; i < page_idx; i++) { 1308 qdf_mem_free_consistent(osdev, osdev->dev, PAGE_SIZE, 1309 dma_pages->page_v_addr_start, 1310 dma_pages->page_p_addr, memctxt); 1311 dma_pages++; 1312 } 1313 qdf_mem_free(pages->dma_pages); 1314 } 1315 1316 out_fail: 1317 pages->cacheable_pages = NULL; 1318 pages->dma_pages = NULL; 1319 pages->num_pages = 0; 1320 return; 1321 } 1322 qdf_export_symbol(qdf_mem_multi_pages_alloc); 1323 1324 /** 1325 * qdf_mem_multi_pages_free() - free large size of kernel memory 1326 * @osdev: OS device handle pointer 1327 * @pages: Multi page information storage 1328 * @memctxt: Memory context 1329 * @cacheable: Coherent memory or cacheable memory 1330 * 1331 * This function will free large size of memory over multiple pages. 1332 * 1333 * Return: None 1334 */ 1335 void qdf_mem_multi_pages_free(qdf_device_t osdev, 1336 struct qdf_mem_multi_page_t *pages, 1337 qdf_dma_context_t memctxt, bool cacheable) 1338 { 1339 unsigned int page_idx; 1340 struct qdf_mem_dma_page_t *dma_pages; 1341 1342 if (cacheable) { 1343 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) 1344 qdf_mem_free(pages->cacheable_pages[page_idx]); 1345 qdf_mem_free(pages->cacheable_pages); 1346 } else { 1347 dma_pages = pages->dma_pages; 1348 for (page_idx = 0; page_idx < pages->num_pages; page_idx++) { 1349 qdf_mem_free_consistent(osdev, osdev->dev, PAGE_SIZE, 1350 dma_pages->page_v_addr_start, 1351 dma_pages->page_p_addr, memctxt); 1352 dma_pages++; 1353 } 1354 qdf_mem_free(pages->dma_pages); 1355 } 1356 1357 pages->cacheable_pages = NULL; 1358 pages->dma_pages = NULL; 1359 pages->num_pages = 0; 1360 return; 1361 } 1362 qdf_export_symbol(qdf_mem_multi_pages_free); 1363 1364 /** 1365 * qdf_mem_multi_page_link() - Make links for multi page elements 1366 * @osdev: OS device handle pointer 1367 * @pages: Multi page information storage 1368 * @elem_size: Single element size 1369 * @elem_count: elements count should be linked 1370 * @cacheable: Coherent memory or cacheable memory 1371 * 1372 * This function will make links for multi page allocated structure 1373 * 1374 * Return: 0 success 1375 */ 1376 int qdf_mem_multi_page_link(qdf_device_t osdev, 1377 struct qdf_mem_multi_page_t *pages, 1378 uint32_t elem_size, uint32_t elem_count, uint8_t cacheable) 1379 { 1380 uint16_t i, i_int; 1381 void *page_info; 1382 void **c_elem = NULL; 1383 uint32_t num_link = 0; 1384 1385 for (i = 0; i < pages->num_pages; i++) { 1386 if (cacheable) 1387 page_info = pages->cacheable_pages[i]; 1388 else 1389 page_info = pages->dma_pages[i].page_v_addr_start; 1390 1391 if (!page_info) 1392 return -ENOMEM; 1393 1394 c_elem = (void **)page_info; 1395 for (i_int = 0; i_int < pages->num_element_per_page; i_int++) { 1396 if (i_int == (pages->num_element_per_page - 1)) { 1397 if (cacheable) 1398 *c_elem = pages-> 1399 cacheable_pages[i + 1]; 1400 else 1401 *c_elem = pages-> 1402 dma_pages[i + 1]. 1403 page_v_addr_start; 1404 num_link++; 1405 break; 1406 } else { 1407 *c_elem = 1408 (void *)(((char *)c_elem) + elem_size); 1409 } 1410 num_link++; 1411 c_elem = (void **)*c_elem; 1412 1413 /* Last link established exit */ 1414 if (num_link == (elem_count - 1)) 1415 break; 1416 } 1417 } 1418 1419 if (c_elem) 1420 *c_elem = NULL; 1421 1422 return 0; 1423 } 1424 qdf_export_symbol(qdf_mem_multi_page_link); 1425 1426 /** 1427 * qdf_mem_copy() - copy memory 1428 * @dst_addr: Pointer to destination memory location (to copy to) 1429 * @src_addr: Pointer to source memory location (to copy from) 1430 * @num_bytes: Number of bytes to copy. 1431 * 1432 * Copy host memory from one location to another, similar to memcpy in 1433 * standard C. Note this function does not specifically handle overlapping 1434 * source and destination memory locations. Calling this function with 1435 * overlapping source and destination memory locations will result in 1436 * unpredictable results. Use qdf_mem_move() if the memory locations 1437 * for the source and destination are overlapping (or could be overlapping!) 1438 * 1439 * Return: none 1440 */ 1441 void qdf_mem_copy(void *dst_addr, const void *src_addr, uint32_t num_bytes) 1442 { 1443 if (0 == num_bytes) { 1444 /* special case where dst_addr or src_addr can be NULL */ 1445 return; 1446 } 1447 1448 if ((dst_addr == NULL) || (src_addr == NULL)) { 1449 QDF_TRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, 1450 "%s called with NULL parameter, source:%pK destination:%pK", 1451 __func__, src_addr, dst_addr); 1452 QDF_ASSERT(0); 1453 return; 1454 } 1455 memcpy(dst_addr, src_addr, num_bytes); 1456 } 1457 qdf_export_symbol(qdf_mem_copy); 1458 1459 qdf_shared_mem_t *qdf_mem_shared_mem_alloc(qdf_device_t osdev, uint32_t size) 1460 { 1461 qdf_shared_mem_t *shared_mem; 1462 qdf_dma_addr_t dma_addr, paddr; 1463 int ret; 1464 1465 shared_mem = qdf_mem_malloc(sizeof(*shared_mem)); 1466 if (!shared_mem) 1467 return NULL; 1468 1469 shared_mem->vaddr = qdf_mem_alloc_consistent(osdev, osdev->dev, 1470 size, qdf_mem_get_dma_addr_ptr(osdev, 1471 &shared_mem->mem_info)); 1472 if (!shared_mem->vaddr) { 1473 qdf_err("Unable to allocate DMA memory for shared resource"); 1474 qdf_mem_free(shared_mem); 1475 return NULL; 1476 } 1477 1478 qdf_mem_set_dma_size(osdev, &shared_mem->mem_info, size); 1479 size = qdf_mem_get_dma_size(osdev, &shared_mem->mem_info); 1480 1481 qdf_mem_zero(shared_mem->vaddr, size); 1482 dma_addr = qdf_mem_get_dma_addr(osdev, &shared_mem->mem_info); 1483 paddr = qdf_mem_paddr_from_dmaaddr(osdev, dma_addr); 1484 1485 qdf_mem_set_dma_pa(osdev, &shared_mem->mem_info, paddr); 1486 ret = qdf_mem_dma_get_sgtable(osdev->dev, &shared_mem->sgtable, 1487 shared_mem->vaddr, dma_addr, size); 1488 if (ret) { 1489 qdf_err("Unable to get DMA sgtable"); 1490 qdf_mem_free_consistent(osdev, osdev->dev, 1491 shared_mem->mem_info.size, 1492 shared_mem->vaddr, 1493 dma_addr, 1494 qdf_get_dma_mem_context(shared_mem, 1495 memctx)); 1496 qdf_mem_free(shared_mem); 1497 return NULL; 1498 } 1499 1500 qdf_dma_get_sgtable_dma_addr(&shared_mem->sgtable); 1501 1502 return shared_mem; 1503 } 1504 1505 qdf_export_symbol(qdf_mem_shared_mem_alloc); 1506 1507 /** 1508 * qdf_mem_zero() - zero out memory 1509 * @ptr: pointer to memory that will be set to zero 1510 * @num_bytes: number of bytes zero 1511 * 1512 * This function sets the memory location to all zeros, essentially clearing 1513 * the memory. 1514 * 1515 * Return: None 1516 */ 1517 void qdf_mem_zero(void *ptr, uint32_t num_bytes) 1518 { 1519 if (0 == num_bytes) { 1520 /* special case where ptr can be NULL */ 1521 return; 1522 } 1523 1524 if (ptr == NULL) { 1525 QDF_TRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, 1526 "%s called with NULL parameter ptr", __func__); 1527 return; 1528 } 1529 memset(ptr, 0, num_bytes); 1530 } 1531 qdf_export_symbol(qdf_mem_zero); 1532 1533 /** 1534 * qdf_mem_copy_toio() - copy memory 1535 * @dst_addr: Pointer to destination memory location (to copy to) 1536 * @src_addr: Pointer to source memory location (to copy from) 1537 * @num_bytes: Number of bytes to copy. 1538 * 1539 * Return: none 1540 */ 1541 void qdf_mem_copy_toio(void *dst_addr, const void *src_addr, uint32_t num_bytes) 1542 { 1543 if (0 == num_bytes) { 1544 /* special case where dst_addr or src_addr can be NULL */ 1545 return; 1546 } 1547 1548 if ((!dst_addr) || (!src_addr)) { 1549 QDF_TRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, 1550 "%s called with NULL parameter, source:%pK destination:%pK", 1551 __func__, src_addr, dst_addr); 1552 QDF_ASSERT(0); 1553 return; 1554 } 1555 memcpy_toio(dst_addr, src_addr, num_bytes); 1556 } 1557 1558 qdf_export_symbol(qdf_mem_copy_toio); 1559 1560 /** 1561 * qdf_mem_set_io() - set (fill) memory with a specified byte value. 1562 * @ptr: Pointer to memory that will be set 1563 * @value: Byte set in memory 1564 * @num_bytes: Number of bytes to be set 1565 * 1566 * Return: None 1567 */ 1568 void qdf_mem_set_io(void *ptr, uint32_t num_bytes, uint32_t value) 1569 { 1570 if (!ptr) { 1571 qdf_print("%s called with NULL parameter ptr", __func__); 1572 return; 1573 } 1574 memset_io(ptr, value, num_bytes); 1575 } 1576 1577 qdf_export_symbol(qdf_mem_set_io); 1578 1579 /** 1580 * qdf_mem_set() - set (fill) memory with a specified byte value. 1581 * @ptr: Pointer to memory that will be set 1582 * @num_bytes: Number of bytes to be set 1583 * @value: Byte set in memory 1584 * 1585 * WARNING: parameter @num_bytes and @value are swapped comparing with 1586 * standard C function "memset", please ensure correct usage of this function! 1587 * 1588 * Return: None 1589 */ 1590 void qdf_mem_set(void *ptr, uint32_t num_bytes, uint32_t value) 1591 { 1592 if (ptr == NULL) { 1593 qdf_print("%s called with NULL parameter ptr", __func__); 1594 return; 1595 } 1596 memset(ptr, value, num_bytes); 1597 } 1598 qdf_export_symbol(qdf_mem_set); 1599 1600 /** 1601 * qdf_mem_move() - move memory 1602 * @dst_addr: pointer to destination memory location (to move to) 1603 * @src_addr: pointer to source memory location (to move from) 1604 * @num_bytes: number of bytes to move. 1605 * 1606 * Move host memory from one location to another, similar to memmove in 1607 * standard C. Note this function *does* handle overlapping 1608 * source and destination memory locations. 1609 1610 * Return: None 1611 */ 1612 void qdf_mem_move(void *dst_addr, const void *src_addr, uint32_t num_bytes) 1613 { 1614 if (0 == num_bytes) { 1615 /* special case where dst_addr or src_addr can be NULL */ 1616 return; 1617 } 1618 1619 if ((dst_addr == NULL) || (src_addr == NULL)) { 1620 QDF_TRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, 1621 "%s called with NULL parameter, source:%pK destination:%pK", 1622 __func__, src_addr, dst_addr); 1623 QDF_ASSERT(0); 1624 return; 1625 } 1626 memmove(dst_addr, src_addr, num_bytes); 1627 } 1628 qdf_export_symbol(qdf_mem_move); 1629 1630 #if defined(A_SIMOS_DEVHOST) || defined(HIF_SDIO) || defined(HIF_USB) 1631 /** 1632 * qdf_mem_dma_alloc() - allocates memory for dma 1633 * @osdev: OS device handle 1634 * @dev: Pointer to device handle 1635 * @size: Size to be allocated 1636 * @phy_addr: Physical address 1637 * 1638 * Return: pointer of allocated memory or null if memory alloc fails 1639 */ 1640 static inline void *qdf_mem_dma_alloc(qdf_device_t osdev, void *dev, 1641 qdf_size_t size, 1642 qdf_dma_addr_t *phy_addr) 1643 { 1644 void *vaddr; 1645 1646 vaddr = qdf_mem_malloc(size); 1647 *phy_addr = ((uintptr_t) vaddr); 1648 /* using this type conversion to suppress "cast from pointer to integer 1649 * of different size" warning on some platforms 1650 */ 1651 BUILD_BUG_ON(sizeof(*phy_addr) < sizeof(vaddr)); 1652 return vaddr; 1653 } 1654 1655 #elif defined(QCA_WIFI_QCA8074) && defined(BUILD_X86) 1656 #define QCA8074_RAM_BASE 0x50000000 1657 #define QDF_MEM_ALLOC_X86_MAX_RETRIES 10 1658 void *qdf_mem_dma_alloc(qdf_device_t osdev, void *dev, qdf_size_t size, 1659 qdf_dma_addr_t *phy_addr) 1660 { 1661 void *vaddr = NULL; 1662 int i; 1663 1664 *phy_addr = 0; 1665 1666 for (i = 0; i < QDF_MEM_ALLOC_X86_MAX_RETRIES; i++) { 1667 vaddr = dma_alloc_coherent(dev, size, phy_addr, 1668 qdf_mem_malloc_flags()); 1669 1670 if (!vaddr) { 1671 qdf_err("%s failed , size: %zu!", __func__, size); 1672 return NULL; 1673 } 1674 1675 if (*phy_addr >= QCA8074_RAM_BASE) 1676 return vaddr; 1677 1678 dma_free_coherent(dev, size, vaddr, *phy_addr); 1679 } 1680 1681 return NULL; 1682 } 1683 1684 #else 1685 static inline void *qdf_mem_dma_alloc(qdf_device_t osdev, void *dev, 1686 qdf_size_t size, qdf_dma_addr_t *paddr) 1687 { 1688 return dma_alloc_coherent(dev, size, paddr, qdf_mem_malloc_flags()); 1689 } 1690 #endif 1691 1692 #if defined(A_SIMOS_DEVHOST) || defined(HIF_SDIO) || defined(HIF_USB) 1693 static inline void 1694 qdf_mem_dma_free(void *dev, qdf_size_t size, void *vaddr, qdf_dma_addr_t paddr) 1695 { 1696 qdf_mem_free(vaddr); 1697 } 1698 #else 1699 1700 static inline void 1701 qdf_mem_dma_free(void *dev, qdf_size_t size, void *vaddr, qdf_dma_addr_t paddr) 1702 { 1703 dma_free_coherent(dev, size, vaddr, paddr); 1704 } 1705 #endif 1706 1707 #ifdef MEMORY_DEBUG 1708 void *qdf_mem_alloc_consistent_debug(qdf_device_t osdev, void *dev, 1709 qdf_size_t size, qdf_dma_addr_t *paddr, 1710 const char *func, uint32_t line, 1711 void *caller) 1712 { 1713 QDF_STATUS status; 1714 enum qdf_debug_domain current_domain = qdf_debug_domain_get(); 1715 qdf_list_t *mem_list = qdf_mem_dma_list(current_domain); 1716 struct qdf_mem_header *header; 1717 void *vaddr; 1718 1719 if (!size || size > QDF_MEM_MAX_MALLOC) { 1720 qdf_err("Cannot malloc %zu bytes @ %s:%d", size, func, line); 1721 return NULL; 1722 } 1723 1724 vaddr = qdf_mem_dma_alloc(osdev, dev, size + QDF_DMA_MEM_DEBUG_SIZE, 1725 paddr); 1726 1727 if (!vaddr) { 1728 qdf_warn("Failed to malloc %zuB @ %s:%d", size, func, line); 1729 return NULL; 1730 } 1731 1732 header = qdf_mem_dma_get_header(vaddr, size); 1733 /* For DMA buffers we only add trailers, this function will init 1734 * the header structure at the tail 1735 * Prefix the header into DMA buffer causes SMMU faults, so 1736 * do not prefix header into the DMA buffers 1737 */ 1738 qdf_mem_header_init(header, size, func, line, caller); 1739 1740 qdf_spin_lock_irqsave(&qdf_mem_dma_list_lock); 1741 status = qdf_list_insert_front(mem_list, &header->node); 1742 qdf_spin_unlock_irqrestore(&qdf_mem_dma_list_lock); 1743 if (QDF_IS_STATUS_ERROR(status)) 1744 qdf_err("Failed to insert memory header; status %d", status); 1745 1746 qdf_mem_dma_inc(size); 1747 1748 return vaddr; 1749 } 1750 qdf_export_symbol(qdf_mem_alloc_consistent_debug); 1751 1752 void qdf_mem_free_consistent_debug(qdf_device_t osdev, void *dev, 1753 qdf_size_t size, void *vaddr, 1754 qdf_dma_addr_t paddr, 1755 qdf_dma_context_t memctx, 1756 const char *func, uint32_t line) 1757 { 1758 enum qdf_debug_domain domain = qdf_debug_domain_get(); 1759 struct qdf_mem_header *header; 1760 enum qdf_mem_validation_bitmap error_bitmap; 1761 1762 /* freeing a null pointer is valid */ 1763 if (qdf_unlikely(!vaddr)) 1764 return; 1765 1766 qdf_talloc_assert_no_children_fl(vaddr, func, line); 1767 1768 qdf_spin_lock_irqsave(&qdf_mem_dma_list_lock); 1769 /* For DMA buffers we only add trailers, this function will retrieve 1770 * the header structure at the tail 1771 * Prefix the header into DMA buffer causes SMMU faults, so 1772 * do not prefix header into the DMA buffers 1773 */ 1774 header = qdf_mem_dma_get_header(vaddr, size); 1775 error_bitmap = qdf_mem_header_validate(header, domain); 1776 if (!error_bitmap) { 1777 header->freed = true; 1778 qdf_list_remove_node(qdf_mem_dma_list(header->domain), 1779 &header->node); 1780 } 1781 qdf_spin_unlock_irqrestore(&qdf_mem_dma_list_lock); 1782 1783 qdf_mem_header_assert_valid(header, domain, error_bitmap, func, line); 1784 1785 qdf_mem_dma_dec(header->size); 1786 qdf_mem_dma_free(dev, size + QDF_DMA_MEM_DEBUG_SIZE, vaddr, paddr); 1787 } 1788 qdf_export_symbol(qdf_mem_free_consistent_debug); 1789 1790 #else 1791 1792 void *qdf_mem_alloc_consistent(qdf_device_t osdev, void *dev, 1793 qdf_size_t size, qdf_dma_addr_t *paddr) 1794 { 1795 void *vaddr = qdf_mem_dma_alloc(osdev, dev, size, paddr); 1796 1797 if (vaddr) 1798 qdf_mem_dma_inc(size); 1799 1800 return vaddr; 1801 } 1802 qdf_export_symbol(qdf_mem_alloc_consistent); 1803 1804 void qdf_mem_free_consistent(qdf_device_t osdev, void *dev, 1805 qdf_size_t size, void *vaddr, 1806 qdf_dma_addr_t paddr, qdf_dma_context_t memctx) 1807 { 1808 qdf_mem_dma_dec(size); 1809 qdf_mem_dma_free(dev, size, vaddr, paddr); 1810 } 1811 qdf_export_symbol(qdf_mem_free_consistent); 1812 1813 #endif /* MEMORY_DEBUG */ 1814 1815 void *qdf_aligned_mem_alloc_consistent_fl( 1816 qdf_device_t osdev, void *dev, qdf_size_t size, 1817 void **vaddr_unaligned, qdf_dma_addr_t *paddr_unaligned, 1818 qdf_dma_addr_t *paddr_aligned, uint32_t ring_base_align, 1819 const char *func, uint32_t line) 1820 { 1821 void *vaddr_aligned; 1822 1823 *vaddr_unaligned = qdf_mem_alloc_consistent(osdev, dev, size, 1824 paddr_unaligned); 1825 if (!*vaddr_unaligned) { 1826 qdf_warn("Failed to alloc %zuB @ %s:%d", size, func, line); 1827 return NULL; 1828 } 1829 1830 if ((unsigned long)(*vaddr_unaligned) % ring_base_align) { 1831 qdf_mem_free_consistent(osdev, dev, size, *vaddr_unaligned, 1832 *paddr_unaligned, 0); 1833 *vaddr_unaligned = qdf_mem_alloc_consistent(osdev, dev, 1834 size + ring_base_align - 1, paddr_unaligned); 1835 if (!*vaddr_unaligned) { 1836 qdf_warn("Failed to alloc %zuB @ %s:%d", 1837 size, func, line); 1838 return NULL; 1839 } 1840 } 1841 1842 vaddr_aligned = *vaddr_unaligned + 1843 ((unsigned long)(*vaddr_unaligned) % ring_base_align); 1844 *paddr_aligned = *paddr_unaligned + ((unsigned long)(vaddr_aligned) - 1845 (unsigned long)(*vaddr_unaligned)); 1846 1847 return vaddr_aligned; 1848 } 1849 qdf_export_symbol(qdf_aligned_mem_alloc_consistent_fl); 1850 1851 /** 1852 * qdf_mem_dma_sync_single_for_device() - assign memory to device 1853 * @osdev: OS device handle 1854 * @bus_addr: dma address to give to the device 1855 * @size: Size of the memory block 1856 * @direction: direction data will be DMAed 1857 * 1858 * Assign memory to the remote device. 1859 * The cache lines are flushed to ram or invalidated as needed. 1860 * 1861 * Return: none 1862 */ 1863 void qdf_mem_dma_sync_single_for_device(qdf_device_t osdev, 1864 qdf_dma_addr_t bus_addr, 1865 qdf_size_t size, 1866 enum dma_data_direction direction) 1867 { 1868 dma_sync_single_for_device(osdev->dev, bus_addr, size, direction); 1869 } 1870 qdf_export_symbol(qdf_mem_dma_sync_single_for_device); 1871 1872 /** 1873 * qdf_mem_dma_sync_single_for_cpu() - assign memory to CPU 1874 * @osdev: OS device handle 1875 * @bus_addr: dma address to give to the cpu 1876 * @size: Size of the memory block 1877 * @direction: direction data will be DMAed 1878 * 1879 * Assign memory to the CPU. 1880 * 1881 * Return: none 1882 */ 1883 void qdf_mem_dma_sync_single_for_cpu(qdf_device_t osdev, 1884 qdf_dma_addr_t bus_addr, 1885 qdf_size_t size, 1886 enum dma_data_direction direction) 1887 { 1888 dma_sync_single_for_cpu(osdev->dev, bus_addr, size, direction); 1889 } 1890 qdf_export_symbol(qdf_mem_dma_sync_single_for_cpu); 1891 1892 void qdf_mem_init(void) 1893 { 1894 qdf_mem_debug_init(); 1895 qdf_net_buf_debug_init(); 1896 qdf_mem_debugfs_init(); 1897 qdf_mem_debug_debugfs_init(); 1898 } 1899 qdf_export_symbol(qdf_mem_init); 1900 1901 void qdf_mem_exit(void) 1902 { 1903 qdf_mem_debug_debugfs_exit(); 1904 qdf_mem_debugfs_exit(); 1905 qdf_net_buf_debug_exit(); 1906 qdf_mem_debug_exit(); 1907 } 1908 qdf_export_symbol(qdf_mem_exit); 1909 1910 /** 1911 * qdf_ether_addr_copy() - copy an Ethernet address 1912 * 1913 * @dst_addr: A six-byte array Ethernet address destination 1914 * @src_addr: A six-byte array Ethernet address source 1915 * 1916 * Please note: dst & src must both be aligned to u16. 1917 * 1918 * Return: none 1919 */ 1920 void qdf_ether_addr_copy(void *dst_addr, const void *src_addr) 1921 { 1922 if ((dst_addr == NULL) || (src_addr == NULL)) { 1923 QDF_TRACE(QDF_MODULE_ID_QDF, QDF_TRACE_LEVEL_ERROR, 1924 "%s called with NULL parameter, source:%pK destination:%pK", 1925 __func__, src_addr, dst_addr); 1926 QDF_ASSERT(0); 1927 return; 1928 } 1929 ether_addr_copy(dst_addr, src_addr); 1930 } 1931 qdf_export_symbol(qdf_ether_addr_copy); 1932 1933