1 /* 2 * Copyright (c) 2015-2019 The Linux Foundation. All rights reserved. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for 5 * any purpose with or without fee is hereby granted, provided that the 6 * above copyright notice and this permission notice appear in all 7 * copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 10 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 11 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 12 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 13 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 14 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 15 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 * PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /** 20 * DOC: hif_napi.c 21 * 22 * HIF NAPI interface implementation 23 */ 24 25 #include <linux/string.h> /* memset */ 26 27 /* Linux headers */ 28 #include <linux/cpumask.h> 29 #include <linux/cpufreq.h> 30 #include <linux/cpu.h> 31 #include <linux/topology.h> 32 #include <linux/interrupt.h> 33 #include <linux/irq.h> 34 #ifdef CONFIG_SCHED_CORE_CTL 35 #include <linux/sched/core_ctl.h> 36 #endif 37 #include <pld_common.h> 38 #include <linux/pm.h> 39 40 /* Driver headers */ 41 #include <hif_napi.h> 42 #include <hif_debug.h> 43 #include <hif_io32.h> 44 #include <ce_api.h> 45 #include <ce_internal.h> 46 #include <hif_irq_affinity.h> 47 #include "qdf_cpuhp.h" 48 #include "qdf_module.h" 49 #include "qdf_net_if.h" 50 #include "qdf_dev.h" 51 52 enum napi_decision_vector { 53 HIF_NAPI_NOEVENT = 0, 54 HIF_NAPI_INITED = 1, 55 HIF_NAPI_CONF_UP = 2 56 }; 57 #define ENABLE_NAPI_MASK (HIF_NAPI_INITED | HIF_NAPI_CONF_UP) 58 59 #ifdef RECEIVE_OFFLOAD 60 /** 61 * hif_rxthread_napi_poll() - dummy napi poll for rx_thread NAPI 62 * @napi: Rx_thread NAPI 63 * @budget: NAPI BUDGET 64 * 65 * Return: 0 as it is not supposed to be polled at all as it is not scheduled. 66 */ 67 static int hif_rxthread_napi_poll(struct napi_struct *napi, int budget) 68 { 69 HIF_ERROR("This napi_poll should not be polled as we don't schedule it"); 70 QDF_ASSERT(0); 71 return 0; 72 } 73 74 /** 75 * hif_init_rx_thread_napi() - Initialize dummy Rx_thread NAPI 76 * @napii: Handle to napi_info holding rx_thread napi 77 * 78 * Return: None 79 */ 80 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 81 { 82 init_dummy_netdev(&napii->rx_thread_netdev); 83 netif_napi_add(&napii->rx_thread_netdev, &napii->rx_thread_napi, 84 hif_rxthread_napi_poll, 64); 85 napi_enable(&napii->rx_thread_napi); 86 } 87 88 /** 89 * hif_deinit_rx_thread_napi() - Deinitialize dummy Rx_thread NAPI 90 * @napii: Handle to napi_info holding rx_thread napi 91 * 92 * Return: None 93 */ 94 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii) 95 { 96 netif_napi_del(&napii->rx_thread_napi); 97 } 98 #else /* RECEIVE_OFFLOAD */ 99 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 100 { 101 } 102 103 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii) 104 { 105 } 106 #endif 107 108 /** 109 * hif_napi_create() - creates the NAPI structures for a given CE 110 * @hif : pointer to hif context 111 * @pipe_id: the CE id on which the instance will be created 112 * @poll : poll function to be used for this NAPI instance 113 * @budget : budget to be registered with the NAPI instance 114 * @scale : scale factor on the weight (to scaler budget to 1000) 115 * @flags : feature flags 116 * 117 * Description: 118 * Creates NAPI instances. This function is called 119 * unconditionally during initialization. It creates 120 * napi structures through the proper HTC/HIF calls. 121 * The structures are disabled on creation. 122 * Note that for each NAPI instance a separate dummy netdev is used 123 * 124 * Return: 125 * < 0: error 126 * = 0: <should never happen> 127 * > 0: id of the created object (for multi-NAPI, number of objects created) 128 */ 129 int hif_napi_create(struct hif_opaque_softc *hif_ctx, 130 int (*poll)(struct napi_struct *, int), 131 int budget, 132 int scale, 133 uint8_t flags) 134 { 135 int i; 136 struct qca_napi_data *napid; 137 struct qca_napi_info *napii; 138 struct CE_state *ce_state; 139 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 140 int rc = 0; 141 142 NAPI_DEBUG("-->(budget=%d, scale=%d)", 143 budget, scale); 144 NAPI_DEBUG("hif->napi_data.state = 0x%08x", 145 hif->napi_data.state); 146 NAPI_DEBUG("hif->napi_data.ce_map = 0x%08x", 147 hif->napi_data.ce_map); 148 149 napid = &(hif->napi_data); 150 if (0 == (napid->state & HIF_NAPI_INITED)) { 151 memset(napid, 0, sizeof(struct qca_napi_data)); 152 qdf_spinlock_create(&(napid->lock)); 153 154 napid->state |= HIF_NAPI_INITED; 155 napid->flags = flags; 156 157 rc = hif_napi_cpu_init(hif_ctx); 158 if (rc != 0 && rc != -EALREADY) { 159 HIF_ERROR("NAPI_initialization failed,. %d", rc); 160 rc = napid->ce_map; 161 goto hnc_err; 162 } else 163 rc = 0; 164 165 HIF_DBG("%s: NAPI structures initialized, rc=%d", 166 __func__, rc); 167 } 168 for (i = 0; i < hif->ce_count; i++) { 169 ce_state = hif->ce_id_to_state[i]; 170 NAPI_DEBUG("ce %d: htt_rx=%d htt_tx=%d", 171 i, ce_state->htt_rx_data, 172 ce_state->htt_tx_data); 173 if (ce_srng_based(hif)) 174 continue; 175 176 if (!ce_state->htt_rx_data) 177 continue; 178 179 /* Now this is a CE where we need NAPI on */ 180 NAPI_DEBUG("Creating NAPI on pipe %d", i); 181 napii = qdf_mem_malloc(sizeof(*napii)); 182 napid->napis[i] = napii; 183 if (!napii) { 184 NAPI_DEBUG("NAPI alloc failure %d", i); 185 rc = -ENOMEM; 186 goto napii_free; 187 } 188 } 189 190 for (i = 0; i < hif->ce_count; i++) { 191 napii = napid->napis[i]; 192 if (!napii) 193 continue; 194 195 NAPI_DEBUG("initializing NAPI for pipe %d", i); 196 memset(napii, 0, sizeof(struct qca_napi_info)); 197 napii->scale = scale; 198 napii->id = NAPI_PIPE2ID(i); 199 napii->hif_ctx = hif_ctx; 200 napii->irq = pld_get_irq(hif->qdf_dev->dev, i); 201 202 if (napii->irq < 0) 203 HIF_WARN("%s: bad IRQ value for CE %d: %d", 204 __func__, i, napii->irq); 205 206 init_dummy_netdev(&(napii->netdev)); 207 208 NAPI_DEBUG("adding napi=%pK to netdev=%pK (poll=%pK, bdgt=%d)", 209 &(napii->napi), &(napii->netdev), poll, budget); 210 netif_napi_add(&(napii->netdev), &(napii->napi), poll, budget); 211 212 NAPI_DEBUG("after napi_add"); 213 NAPI_DEBUG("napi=0x%pK, netdev=0x%pK", 214 &(napii->napi), &(napii->netdev)); 215 NAPI_DEBUG("napi.dev_list.prev=0x%pK, next=0x%pK", 216 napii->napi.dev_list.prev, 217 napii->napi.dev_list.next); 218 NAPI_DEBUG("dev.napi_list.prev=0x%pK, next=0x%pK", 219 napii->netdev.napi_list.prev, 220 napii->netdev.napi_list.next); 221 222 hif_init_rx_thread_napi(napii); 223 napii->lro_ctx = qdf_lro_init(); 224 NAPI_DEBUG("Registering LRO for ce_id %d NAPI callback for %d lro_ctx %pK\n", 225 i, napii->id, napii->lro_ctx); 226 227 /* It is OK to change the state variable below without 228 * protection as there should be no-one around yet 229 */ 230 napid->ce_map |= (0x01 << i); 231 HIF_DBG("%s: NAPI id %d created for pipe %d", __func__, 232 napii->id, i); 233 } 234 235 /* no ces registered with the napi */ 236 if (!ce_srng_based(hif) && napid->ce_map == 0) { 237 HIF_WARN("%s: no napis created for copy engines", __func__); 238 rc = -EFAULT; 239 goto napii_free; 240 } 241 242 NAPI_DEBUG("napi map = %x", napid->ce_map); 243 NAPI_DEBUG("NAPI ids created for all applicable pipes"); 244 return napid->ce_map; 245 246 napii_free: 247 for (i = 0; i < hif->ce_count; i++) { 248 napii = napid->napis[i]; 249 napid->napis[i] = NULL; 250 if (napii) 251 qdf_mem_free(napii); 252 } 253 254 hnc_err: 255 NAPI_DEBUG("<--napi_instances_map=%x]", napid->ce_map); 256 return rc; 257 } 258 qdf_export_symbol(hif_napi_create); 259 260 #ifdef RECEIVE_OFFLOAD 261 void hif_napi_rx_offld_flush_cb_register(struct hif_opaque_softc *hif_hdl, 262 void (offld_flush_handler)(void *)) 263 { 264 int i; 265 struct CE_state *ce_state; 266 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 267 struct qca_napi_data *napid; 268 struct qca_napi_info *napii; 269 270 if (!scn) { 271 HIF_ERROR("%s: hif_state NULL!", __func__); 272 QDF_ASSERT(0); 273 return; 274 } 275 276 napid = hif_napi_get_all(hif_hdl); 277 for (i = 0; i < scn->ce_count; i++) { 278 ce_state = scn->ce_id_to_state[i]; 279 if (ce_state && (ce_state->htt_rx_data)) { 280 napii = napid->napis[i]; 281 napii->offld_flush_cb = offld_flush_handler; 282 HIF_DBG("Registering offload for ce_id %d NAPI callback for %d flush_cb %pK\n", 283 i, napii->id, napii->offld_flush_cb); 284 } 285 } 286 } 287 288 void hif_napi_rx_offld_flush_cb_deregister(struct hif_opaque_softc *hif_hdl) 289 { 290 int i; 291 struct CE_state *ce_state; 292 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 293 struct qca_napi_data *napid; 294 struct qca_napi_info *napii; 295 296 if (!scn) { 297 HIF_ERROR("%s: hif_state NULL!", __func__); 298 QDF_ASSERT(0); 299 return; 300 } 301 302 napid = hif_napi_get_all(hif_hdl); 303 for (i = 0; i < scn->ce_count; i++) { 304 ce_state = scn->ce_id_to_state[i]; 305 if (ce_state && (ce_state->htt_rx_data)) { 306 napii = napid->napis[i]; 307 HIF_DBG("deRegistering offld for ce_id %d NAPI callback for %d flush_cb %pK\n", 308 i, napii->id, napii->offld_flush_cb); 309 /* Not required */ 310 napii->offld_flush_cb = NULL; 311 } 312 } 313 } 314 #endif /* RECEIVE_OFFLOAD */ 315 316 /** 317 * 318 * hif_napi_destroy() - destroys the NAPI structures for a given instance 319 * @hif : pointer to hif context 320 * @ce_id : the CE id whose napi instance will be destroyed 321 * @force : if set, will destroy even if entry is active (de-activates) 322 * 323 * Description: 324 * Destroy a given NAPI instance. This function is called 325 * unconditionally during cleanup. 326 * Refuses to destroy an entry of it is still enabled (unless force=1) 327 * Marks the whole napi_data invalid if all instances are destroyed. 328 * 329 * Return: 330 * -EINVAL: specific entry has not been created 331 * -EPERM : specific entry is still active 332 * 0 < : error 333 * 0 = : success 334 */ 335 int hif_napi_destroy(struct hif_opaque_softc *hif_ctx, 336 uint8_t id, 337 int force) 338 { 339 uint8_t ce = NAPI_ID2PIPE(id); 340 int rc = 0; 341 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 342 343 NAPI_DEBUG("-->(id=%d, force=%d)", id, force); 344 345 if (0 == (hif->napi_data.state & HIF_NAPI_INITED)) { 346 HIF_ERROR("%s: NAPI not initialized or entry %d not created", 347 __func__, id); 348 rc = -EINVAL; 349 } else if (0 == (hif->napi_data.ce_map & (0x01 << ce))) { 350 HIF_ERROR("%s: NAPI instance %d (pipe %d) not created", 351 __func__, id, ce); 352 if (hif->napi_data.napis[ce]) 353 HIF_ERROR("%s: memory allocated but ce_map not set %d (pipe %d)", 354 __func__, id, ce); 355 rc = -EINVAL; 356 } else { 357 struct qca_napi_data *napid; 358 struct qca_napi_info *napii; 359 360 napid = &(hif->napi_data); 361 napii = napid->napis[ce]; 362 if (!napii) { 363 if (napid->ce_map & (0x01 << ce)) 364 HIF_ERROR("%s: napii & ce_map out of sync(ce %d)", 365 __func__, ce); 366 return -EINVAL; 367 } 368 369 370 if (hif->napi_data.state == HIF_NAPI_CONF_UP) { 371 if (force) { 372 napi_disable(&(napii->napi)); 373 HIF_DBG("%s: NAPI entry %d force disabled", 374 __func__, id); 375 NAPI_DEBUG("NAPI %d force disabled", id); 376 } else { 377 HIF_ERROR("%s: Cannot destroy active NAPI %d", 378 __func__, id); 379 rc = -EPERM; 380 } 381 } 382 if (0 == rc) { 383 NAPI_DEBUG("before napi_del"); 384 NAPI_DEBUG("napi.dlist.prv=0x%pK, next=0x%pK", 385 napii->napi.dev_list.prev, 386 napii->napi.dev_list.next); 387 NAPI_DEBUG("dev.napi_l.prv=0x%pK, next=0x%pK", 388 napii->netdev.napi_list.prev, 389 napii->netdev.napi_list.next); 390 391 qdf_lro_deinit(napii->lro_ctx); 392 netif_napi_del(&(napii->napi)); 393 hif_deinit_rx_thread_napi(napii); 394 395 napid->ce_map &= ~(0x01 << ce); 396 napid->napis[ce] = NULL; 397 napii->scale = 0; 398 qdf_mem_free(napii); 399 HIF_DBG("%s: NAPI %d destroyed\n", __func__, id); 400 401 /* if there are no active instances and 402 * if they are all destroyed, 403 * set the whole structure to uninitialized state 404 */ 405 if (napid->ce_map == 0) { 406 rc = hif_napi_cpu_deinit(hif_ctx); 407 /* caller is tolerant to receiving !=0 rc */ 408 409 qdf_spinlock_destroy(&(napid->lock)); 410 memset(napid, 411 0, sizeof(struct qca_napi_data)); 412 HIF_DBG("%s: no NAPI instances. Zapped.", 413 __func__); 414 } 415 } 416 } 417 418 return rc; 419 } 420 qdf_export_symbol(hif_napi_destroy); 421 422 #ifdef FEATURE_LRO 423 void *hif_napi_get_lro_info(struct hif_opaque_softc *hif_hdl, int napi_id) 424 { 425 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 426 struct qca_napi_data *napid; 427 struct qca_napi_info *napii; 428 429 napid = &(scn->napi_data); 430 napii = napid->napis[NAPI_ID2PIPE(napi_id)]; 431 432 if (napii) 433 return napii->lro_ctx; 434 return 0; 435 } 436 #endif 437 438 /** 439 * 440 * hif_napi_get_all() - returns the address of the whole HIF NAPI structure 441 * @hif: pointer to hif context 442 * 443 * Description: 444 * Returns the address of the whole structure 445 * 446 * Return: 447 * <addr>: address of the whole HIF NAPI structure 448 */ 449 inline struct qca_napi_data *hif_napi_get_all(struct hif_opaque_softc *hif_ctx) 450 { 451 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 452 453 return &(hif->napi_data); 454 } 455 456 struct qca_napi_info *hif_get_napi(int napi_id, struct qca_napi_data *napid) 457 { 458 int id = NAPI_ID2PIPE(napi_id); 459 460 return napid->napis[id]; 461 } 462 463 /** 464 * 465 * hif_napi_event() - reacts to events that impact NAPI 466 * @hif : pointer to hif context 467 * @evnt: event that has been detected 468 * @data: more data regarding the event 469 * 470 * Description: 471 * This function handles two types of events: 472 * 1- Events that change the state of NAPI (enabled/disabled): 473 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE} 474 * The state is retrievable by "hdd_napi_enabled(-1)" 475 * - NAPI will be on if either INI file is on and it has not been disabled 476 * by a subsequent vendor CMD, 477 * or it has been enabled by a vendor CMD. 478 * 2- Events that change the CPU affinity of a NAPI instance/IRQ: 479 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE} 480 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode 481 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy() 482 * - In LO tput mode, NAPI will yield control if its interrupts to the system 483 * management functions. However in HI throughput mode, NAPI will actively 484 * manage its interrupts/instances (by trying to disperse them out to 485 * separate performance cores). 486 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events. 487 * 488 * + In some cases (roaming peer management is the only case so far), a 489 * a client can trigger a "SERIALIZE" event. Basically, this means that the 490 * users is asking NAPI to go into a truly single execution context state. 491 * So, NAPI indicates to msm-irqbalancer that it wants to be blacklisted, 492 * (if called for the first time) and then moves all IRQs (for NAPI 493 * instances) to be collapsed to a single core. If called multiple times, 494 * it will just re-collapse the CPUs. This is because blacklist-on() API 495 * is reference-counted, and because the API has already been called. 496 * 497 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go 498 * to its "normal" operation. Optionally, they can give a timeout value (in 499 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this 500 * case, NAPI will just set the current throughput state to uninitialized 501 * and set the delay period. Once policy handler is called, it would skip 502 * applying the policy delay period times, and otherwise apply the policy. 503 * 504 * Return: 505 * < 0: some error 506 * = 0: event handled successfully 507 */ 508 int hif_napi_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event, 509 void *data) 510 { 511 int rc = 0; 512 uint32_t prev_state; 513 int i; 514 bool state_changed; 515 struct napi_struct *napi; 516 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 517 struct qca_napi_data *napid = &(hif->napi_data); 518 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED; 519 enum { 520 BLACKLIST_NOT_PENDING, 521 BLACKLIST_ON_PENDING, 522 BLACKLIST_OFF_PENDING 523 } blacklist_pending = BLACKLIST_NOT_PENDING; 524 525 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data); 526 527 if (ce_srng_based(hif)) 528 return hif_exec_event(hif_ctx, event, data); 529 530 if ((napid->state & HIF_NAPI_INITED) == 0) { 531 NAPI_DEBUG("%s: got event when NAPI not initialized", 532 __func__); 533 return -EINVAL; 534 } 535 qdf_spin_lock_bh(&(napid->lock)); 536 prev_state = napid->state; 537 switch (event) { 538 case NAPI_EVT_INI_FILE: 539 case NAPI_EVT_CMD_STATE: 540 case NAPI_EVT_INT_STATE: { 541 int on = (data != ((void *)0)); 542 543 HIF_DBG("%s: recved evnt: STATE_CMD %d; v = %d (state=0x%0x)", 544 __func__, event, 545 on, prev_state); 546 if (on) 547 if (prev_state & HIF_NAPI_CONF_UP) { 548 HIF_DBG("%s: duplicate NAPI conf ON msg", 549 __func__); 550 } else { 551 HIF_DBG("%s: setting state to ON", 552 __func__); 553 napid->state |= HIF_NAPI_CONF_UP; 554 } 555 else /* off request */ 556 if (prev_state & HIF_NAPI_CONF_UP) { 557 HIF_DBG("%s: setting state to OFF", 558 __func__); 559 napid->state &= ~HIF_NAPI_CONF_UP; 560 } else { 561 HIF_DBG("%s: duplicate NAPI conf OFF msg", 562 __func__); 563 } 564 break; 565 } 566 /* case NAPI_INIT_FILE/CMD_STATE */ 567 568 case NAPI_EVT_CPU_STATE: { 569 int cpu = ((unsigned long int)data >> 16); 570 int val = ((unsigned long int)data & 0x0ff); 571 572 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d", 573 __func__, cpu, val); 574 575 /* state has already been set by hnc_cpu_notify_cb */ 576 if ((val == QCA_NAPI_CPU_DOWN) && 577 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */ 578 (napid->napi_cpu[cpu].napis != 0)) { 579 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d", 580 __func__, cpu); 581 rc = hif_napi_cpu_migrate(napid, 582 cpu, 583 HNC_ACT_RELOCATE); 584 napid->napi_cpu[cpu].napis = 0; 585 } 586 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */ 587 break; 588 } 589 590 case NAPI_EVT_TPUT_STATE: { 591 tput_mode = (enum qca_napi_tput_state)data; 592 if (tput_mode == QCA_NAPI_TPUT_LO) { 593 /* from TPUT_HI -> TPUT_LO */ 594 NAPI_DEBUG("%s: Moving to napi_tput_LO state", 595 __func__); 596 blacklist_pending = BLACKLIST_OFF_PENDING; 597 /* 598 * Ideally we should "collapse" interrupts here, since 599 * we are "dispersing" interrupts in the "else" case. 600 * This allows the possibility that our interrupts may 601 * still be on the perf cluster the next time we enter 602 * high tput mode. However, the irq_balancer is free 603 * to move our interrupts to power cluster once 604 * blacklisting has been turned off in the "else" case. 605 */ 606 } else { 607 /* from TPUT_LO -> TPUT->HI */ 608 NAPI_DEBUG("%s: Moving to napi_tput_HI state", 609 __func__); 610 rc = hif_napi_cpu_migrate(napid, 611 HNC_ANY_CPU, 612 HNC_ACT_DISPERSE); 613 614 blacklist_pending = BLACKLIST_ON_PENDING; 615 } 616 napid->napi_mode = tput_mode; 617 break; 618 } 619 620 case NAPI_EVT_USR_SERIAL: { 621 unsigned long users = (unsigned long)data; 622 623 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld", 624 __func__, users); 625 626 rc = hif_napi_cpu_migrate(napid, 627 HNC_ANY_CPU, 628 HNC_ACT_COLLAPSE); 629 if ((users == 0) && (rc == 0)) 630 blacklist_pending = BLACKLIST_ON_PENDING; 631 break; 632 } 633 case NAPI_EVT_USR_NORMAL: { 634 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__); 635 if (!napid->user_cpu_affin_mask) 636 blacklist_pending = BLACKLIST_OFF_PENDING; 637 /* 638 * Deserialization timeout is handled at hdd layer; 639 * just mark current mode to uninitialized to ensure 640 * it will be set when the delay is over 641 */ 642 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED; 643 break; 644 } 645 default: { 646 HIF_ERROR("%s: unknown event: %d (data=0x%0lx)", 647 __func__, event, (unsigned long) data); 648 break; 649 } /* default */ 650 }; /* switch */ 651 652 653 switch (blacklist_pending) { 654 case BLACKLIST_ON_PENDING: 655 /* assume the control of WLAN IRQs */ 656 hif_napi_cpu_blacklist(napid, BLACKLIST_ON); 657 break; 658 case BLACKLIST_OFF_PENDING: 659 /* yield the control of WLAN IRQs */ 660 hif_napi_cpu_blacklist(napid, BLACKLIST_OFF); 661 break; 662 default: /* nothing to do */ 663 break; 664 } /* switch blacklist_pending */ 665 666 /* we want to perform the comparison in lock: 667 * there is a possiblity of hif_napi_event get called 668 * from two different contexts (driver unload and cpu hotplug 669 * notification) and napid->state get changed 670 * in driver unload context and can lead to race condition 671 * in cpu hotplug context. Therefore, perform the napid->state 672 * comparison before releasing lock. 673 */ 674 state_changed = (prev_state != napid->state); 675 qdf_spin_unlock_bh(&(napid->lock)); 676 677 if (state_changed) { 678 if (napid->state == ENABLE_NAPI_MASK) { 679 rc = 1; 680 for (i = 0; i < CE_COUNT_MAX; i++) { 681 struct qca_napi_info *napii = napid->napis[i]; 682 if (napii) { 683 napi = &(napii->napi); 684 NAPI_DEBUG("%s: enabling NAPI %d", 685 __func__, i); 686 napi_enable(napi); 687 } 688 } 689 } else { 690 rc = 0; 691 for (i = 0; i < CE_COUNT_MAX; i++) { 692 struct qca_napi_info *napii = napid->napis[i]; 693 if (napii) { 694 napi = &(napii->napi); 695 NAPI_DEBUG("%s: disabling NAPI %d", 696 __func__, i); 697 napi_disable(napi); 698 /* in case it is affined, remove it */ 699 qdf_dev_set_irq_affinity(napii->irq, 700 NULL); 701 } 702 } 703 } 704 } else { 705 HIF_DBG("%s: no change in hif napi state (still %d)", 706 __func__, prev_state); 707 } 708 709 NAPI_DEBUG("<--[rc=%d]", rc); 710 return rc; 711 } 712 qdf_export_symbol(hif_napi_event); 713 714 /** 715 * hif_napi_enabled() - checks whether NAPI is enabled for given ce or not 716 * @hif: hif context 717 * @ce : CE instance (or -1, to check if any CEs are enabled) 718 * 719 * Return: bool 720 */ 721 int hif_napi_enabled(struct hif_opaque_softc *hif_ctx, int ce) 722 { 723 int rc; 724 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 725 726 if (-1 == ce) 727 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK)); 728 else 729 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK) && 730 (hif->napi_data.ce_map & (0x01 << ce))); 731 return rc; 732 } 733 qdf_export_symbol(hif_napi_enabled); 734 735 /** 736 * hif_napi_created() - checks whether NAPI is created for given ce or not 737 * @hif: hif context 738 * @ce : CE instance 739 * 740 * Return: bool 741 */ 742 bool hif_napi_created(struct hif_opaque_softc *hif_ctx, int ce) 743 { 744 int rc; 745 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 746 747 rc = (hif->napi_data.ce_map & (0x01 << ce)); 748 749 return !!rc; 750 } 751 qdf_export_symbol(hif_napi_created); 752 753 /** 754 * hif_napi_enable_irq() - enables bus interrupts after napi_complete 755 * 756 * @hif: hif context 757 * @id : id of NAPI instance calling this (used to determine the CE) 758 * 759 * Return: void 760 */ 761 inline void hif_napi_enable_irq(struct hif_opaque_softc *hif, int id) 762 { 763 struct hif_softc *scn = HIF_GET_SOFTC(hif); 764 765 hif_irq_enable(scn, NAPI_ID2PIPE(id)); 766 } 767 768 769 /** 770 * hif_napi_schedule() - schedules napi, updates stats 771 * @scn: hif context 772 * @ce_id: index of napi instance 773 * 774 * Return: false if napi didn't enable or already scheduled, otherwise true 775 */ 776 bool hif_napi_schedule(struct hif_opaque_softc *hif_ctx, int ce_id) 777 { 778 int cpu = smp_processor_id(); 779 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 780 struct qca_napi_info *napii; 781 782 napii = scn->napi_data.napis[ce_id]; 783 if (qdf_unlikely(!napii)) { 784 HIF_ERROR("%s, scheduling unallocated napi (ce:%d)", 785 __func__, ce_id); 786 qdf_atomic_dec(&scn->active_tasklet_cnt); 787 return false; 788 } 789 790 if (test_bit(NAPI_STATE_SCHED, &napii->napi.state)) { 791 NAPI_DEBUG("napi scheduled, return"); 792 qdf_atomic_dec(&scn->active_tasklet_cnt); 793 return false; 794 } 795 796 hif_record_ce_desc_event(scn, ce_id, NAPI_SCHEDULE, 797 NULL, NULL, 0, 0); 798 napii->stats[cpu].napi_schedules++; 799 NAPI_DEBUG("scheduling napi %d (ce:%d)", napii->id, ce_id); 800 napi_schedule(&(napii->napi)); 801 802 return true; 803 } 804 qdf_export_symbol(hif_napi_schedule); 805 806 /** 807 * hif_napi_correct_cpu() - correct the interrupt affinity for napi if needed 808 * @napi_info: pointer to qca_napi_info for the napi instance 809 * 810 * Return: true => interrupt already on correct cpu, no correction needed 811 * false => interrupt on wrong cpu, correction done for cpu affinity 812 * of the interrupt 813 */ 814 static inline 815 bool hif_napi_correct_cpu(struct qca_napi_info *napi_info) 816 { 817 bool right_cpu = true; 818 int rc = 0; 819 int cpu; 820 struct qca_napi_data *napid; 821 QDF_STATUS ret; 822 823 napid = hif_napi_get_all(GET_HIF_OPAQUE_HDL(napi_info->hif_ctx)); 824 825 if (napid->flags & QCA_NAPI_FEATURE_CPU_CORRECTION) { 826 827 cpu = qdf_get_cpu(); 828 if (unlikely((hif_napi_cpu_blacklist(napid, 829 BLACKLIST_QUERY) > 0) && 830 (cpu != napi_info->cpu))) { 831 right_cpu = false; 832 833 NAPI_DEBUG("interrupt on wrong CPU, correcting"); 834 napi_info->cpumask.bits[0] = (0x01 << napi_info->cpu); 835 836 irq_modify_status(napi_info->irq, IRQ_NO_BALANCING, 0); 837 ret = qdf_dev_set_irq_affinity(napi_info->irq, 838 (struct qdf_cpu_mask *) 839 &napi_info->cpumask); 840 rc = qdf_status_to_os_return(ret); 841 irq_modify_status(napi_info->irq, 0, IRQ_NO_BALANCING); 842 843 if (rc) 844 HIF_ERROR("error setting irq affinity hint: %d", 845 rc); 846 else 847 napi_info->stats[cpu].cpu_corrected++; 848 } 849 } 850 return right_cpu; 851 } 852 853 #ifdef RECEIVE_OFFLOAD 854 /** 855 * hif_napi_offld_flush_cb() - Call upper layer flush callback 856 * @napi_info: Handle to hif_napi_info 857 * 858 * Return: None 859 */ 860 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 861 { 862 if (napi_info->offld_flush_cb) 863 napi_info->offld_flush_cb(napi_info); 864 } 865 #else 866 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 867 { 868 } 869 #endif 870 871 /** 872 * hif_napi_poll() - NAPI poll routine 873 * @napi : pointer to NAPI struct as kernel holds it 874 * @budget: 875 * 876 * This is the body of the poll function. 877 * The poll function is called by kernel. So, there is a wrapper 878 * function in HDD, which in turn calls this function. 879 * Two main reasons why the whole thing is not implemented in HDD: 880 * a) references to things like ce_service that HDD is not aware of 881 * b) proximity to the implementation of ce_tasklet, which the body 882 * of this function should be very close to. 883 * 884 * NOTE TO THE MAINTAINER: 885 * Consider this function and ce_tasklet very tightly coupled pairs. 886 * Any changes to ce_tasklet or this function may likely need to be 887 * reflected in the counterpart. 888 * 889 * Returns: 890 * int: the amount of work done in this poll (<= budget) 891 */ 892 int hif_napi_poll(struct hif_opaque_softc *hif_ctx, 893 struct napi_struct *napi, 894 int budget) 895 { 896 int rc = 0; /* default: no work done, also takes care of error */ 897 int normalized = 0; 898 int bucket; 899 int cpu = smp_processor_id(); 900 bool poll_on_right_cpu; 901 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 902 struct qca_napi_info *napi_info; 903 struct CE_state *ce_state = NULL; 904 905 if (unlikely(NULL == hif)) { 906 HIF_ERROR("%s: hif context is NULL", __func__); 907 QDF_ASSERT(0); 908 goto out; 909 } 910 911 napi_info = (struct qca_napi_info *) 912 container_of(napi, struct qca_napi_info, napi); 913 914 NAPI_DEBUG("%s -->(napi(%d, irq=%d), budget=%d)", 915 __func__, napi_info->id, napi_info->irq, budget); 916 917 napi_info->stats[cpu].napi_polls++; 918 919 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 920 NAPI_POLL_ENTER, NULL, NULL, cpu, 0); 921 922 rc = ce_per_engine_service(hif, NAPI_ID2PIPE(napi_info->id)); 923 NAPI_DEBUG("%s: ce_per_engine_service processed %d msgs", 924 __func__, rc); 925 926 hif_napi_offld_flush_cb(napi_info); 927 928 /* do not return 0, if there was some work done, 929 * even if it is below the scale 930 */ 931 if (rc) { 932 napi_info->stats[cpu].napi_workdone += rc; 933 normalized = (rc / napi_info->scale); 934 if (normalized == 0) 935 normalized++; 936 bucket = (normalized - 1) / 937 (QCA_NAPI_BUDGET / QCA_NAPI_NUM_BUCKETS); 938 if (bucket >= QCA_NAPI_NUM_BUCKETS) { 939 bucket = QCA_NAPI_NUM_BUCKETS - 1; 940 HIF_ERROR("Bad bucket#(%d) > QCA_NAPI_NUM_BUCKETS(%d)" 941 " normalized %d, napi budget %d", 942 bucket, QCA_NAPI_NUM_BUCKETS, 943 normalized, QCA_NAPI_BUDGET); 944 } 945 napi_info->stats[cpu].napi_budget_uses[bucket]++; 946 } else { 947 /* if ce_per engine reports 0, then poll should be terminated */ 948 NAPI_DEBUG("%s:%d: nothing processed by CE. Completing NAPI", 949 __func__, __LINE__); 950 } 951 952 ce_state = hif->ce_id_to_state[NAPI_ID2PIPE(napi_info->id)]; 953 954 /* 955 * Not using the API hif_napi_correct_cpu directly in the if statement 956 * below since the API may not get evaluated if put at the end if any 957 * prior condition would evaluate to be true. The CPU correction 958 * check should kick in every poll. 959 */ 960 #ifdef NAPI_YIELD_BUDGET_BASED 961 if (ce_state && (ce_state->force_break || 0 == rc)) { 962 #else 963 poll_on_right_cpu = hif_napi_correct_cpu(napi_info); 964 if ((ce_state) && 965 (!ce_check_rx_pending(ce_state) || (0 == rc) || 966 !poll_on_right_cpu)) { 967 #endif 968 napi_info->stats[cpu].napi_completes++; 969 #ifdef NAPI_YIELD_BUDGET_BASED 970 ce_state->force_break = 0; 971 #endif 972 973 hif_record_ce_desc_event(hif, ce_state->id, NAPI_COMPLETE, 974 NULL, NULL, 0, 0); 975 if (normalized >= budget) 976 normalized = budget - 1; 977 978 napi_complete(napi); 979 /* enable interrupts */ 980 hif_napi_enable_irq(hif_ctx, napi_info->id); 981 /* support suspend/resume */ 982 qdf_atomic_dec(&(hif->active_tasklet_cnt)); 983 984 NAPI_DEBUG("%s:%d: napi_complete + enabling the interrupts", 985 __func__, __LINE__); 986 } else { 987 /* 4.4 kernel NAPI implementation requires drivers to 988 * return full work when they ask to be re-scheduled, 989 * or napi_complete and re-start with a fresh interrupt 990 */ 991 normalized = budget; 992 } 993 994 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 995 NAPI_POLL_EXIT, NULL, NULL, normalized, 0); 996 997 NAPI_DEBUG("%s <--[normalized=%d]", __func__, normalized); 998 return normalized; 999 out: 1000 return rc; 1001 } 1002 qdf_export_symbol(hif_napi_poll); 1003 1004 void hif_update_napi_max_poll_time(struct CE_state *ce_state, 1005 int ce_id, 1006 int cpu_id) 1007 { 1008 struct hif_softc *hif; 1009 struct qca_napi_info *napi_info; 1010 unsigned long long napi_poll_time = sched_clock() - 1011 ce_state->ce_service_start_time; 1012 1013 hif = ce_state->scn; 1014 napi_info = hif->napi_data.napis[ce_id]; 1015 if (napi_poll_time > 1016 napi_info->stats[cpu_id].napi_max_poll_time) 1017 napi_info->stats[cpu_id].napi_max_poll_time = napi_poll_time; 1018 } 1019 qdf_export_symbol(hif_update_napi_max_poll_time); 1020 1021 #ifdef HIF_IRQ_AFFINITY 1022 /** 1023 * 1024 * hif_napi_update_yield_stats() - update NAPI yield related stats 1025 * @cpu_id: CPU ID for which stats needs to be updates 1026 * @ce_id: Copy Engine ID for which yield stats needs to be updates 1027 * @time_limit_reached: indicates whether the time limit was reached 1028 * @rxpkt_thresh_reached: indicates whether rx packet threshold was reached 1029 * 1030 * Return: None 1031 */ 1032 void hif_napi_update_yield_stats(struct CE_state *ce_state, 1033 bool time_limit_reached, 1034 bool rxpkt_thresh_reached) 1035 { 1036 struct hif_softc *hif; 1037 struct qca_napi_data *napi_data = NULL; 1038 int ce_id = 0; 1039 int cpu_id = 0; 1040 1041 if (unlikely(NULL == ce_state)) { 1042 QDF_ASSERT(NULL != ce_state); 1043 return; 1044 } 1045 1046 hif = ce_state->scn; 1047 1048 if (unlikely(NULL == hif)) { 1049 QDF_ASSERT(NULL != hif); 1050 return; 1051 } 1052 napi_data = &(hif->napi_data); 1053 if (unlikely(NULL == napi_data)) { 1054 QDF_ASSERT(NULL != napi_data); 1055 return; 1056 } 1057 1058 ce_id = ce_state->id; 1059 cpu_id = qdf_get_cpu(); 1060 1061 if (unlikely(!napi_data->napis[ce_id])) { 1062 HIF_INFO("%s: NAPI info is NULL for ce id: %d", 1063 __func__, ce_id); 1064 return; 1065 } 1066 1067 if (time_limit_reached) 1068 napi_data->napis[ce_id]->stats[cpu_id].time_limit_reached++; 1069 else 1070 napi_data->napis[ce_id]->stats[cpu_id].rxpkt_thresh_reached++; 1071 1072 hif_update_napi_max_poll_time(ce_state, ce_id, 1073 cpu_id); 1074 } 1075 1076 /** 1077 * 1078 * hif_napi_stats() - display NAPI CPU statistics 1079 * @napid: pointer to qca_napi_data 1080 * 1081 * Description: 1082 * Prints the various CPU cores on which the NAPI instances /CEs interrupts 1083 * are being executed. Can be called from outside NAPI layer. 1084 * 1085 * Return: None 1086 */ 1087 void hif_napi_stats(struct qca_napi_data *napid) 1088 { 1089 int i; 1090 struct qca_napi_cpu *cpu; 1091 1092 if (napid == NULL) { 1093 qdf_debug("%s: napiid struct is null", __func__); 1094 return; 1095 } 1096 1097 cpu = napid->napi_cpu; 1098 qdf_debug("NAPI CPU TABLE"); 1099 qdf_debug("lilclhead=%d, bigclhead=%d", 1100 napid->lilcl_head, napid->bigcl_head); 1101 for (i = 0; i < NR_CPUS; i++) { 1102 qdf_debug("CPU[%02d]: state:%d crid=%02d clid=%02d crmk:0x%0lx thmk:0x%0lx frq:%d napi = 0x%08x lnk:%d", 1103 i, 1104 cpu[i].state, cpu[i].core_id, cpu[i].cluster_id, 1105 cpu[i].core_mask.bits[0], 1106 cpu[i].thread_mask.bits[0], 1107 cpu[i].max_freq, cpu[i].napis, 1108 cpu[i].cluster_nxt); 1109 } 1110 } 1111 1112 #ifdef FEATURE_NAPI_DEBUG 1113 /* 1114 * Local functions 1115 * - no argument checks, all internal/trusted callers 1116 */ 1117 static void hnc_dump_cpus(struct qca_napi_data *napid) 1118 { 1119 hif_napi_stats(napid); 1120 } 1121 #else 1122 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ }; 1123 #endif /* FEATURE_NAPI_DEBUG */ 1124 /** 1125 * hnc_link_clusters() - partitions to cpu table into clusters 1126 * @napid: pointer to NAPI data 1127 * 1128 * Takes in a CPU topology table and builds two linked lists 1129 * (big cluster cores, list-head at bigcl_head, and little cluster 1130 * cores, list-head at lilcl_head) out of it. 1131 * 1132 * If there are more than two clusters: 1133 * - bigcl_head and lilcl_head will be different, 1134 * - the cluster with highest cpufreq will be considered the "big" cluster. 1135 * If there are more than one with the highest frequency, the *last* of such 1136 * clusters will be designated as the "big cluster" 1137 * - the cluster with lowest cpufreq will be considered the "li'l" cluster. 1138 * If there are more than one clusters with the lowest cpu freq, the *first* 1139 * of such clusters will be designated as the "little cluster" 1140 * - We only support up to 32 clusters 1141 * Return: 0 : OK 1142 * !0: error (at least one of lil/big clusters could not be found) 1143 */ 1144 #define HNC_MIN_CLUSTER 0 1145 #define HNC_MAX_CLUSTER 1 1146 static int hnc_link_clusters(struct qca_napi_data *napid) 1147 { 1148 int rc = 0; 1149 1150 int i; 1151 int it = 0; 1152 uint32_t cl_done = 0x0; 1153 int cl, curcl, curclhead = 0; 1154 int more; 1155 unsigned int lilfrq = INT_MAX; 1156 unsigned int bigfrq = 0; 1157 unsigned int clfrq = 0; 1158 int prev = 0; 1159 struct qca_napi_cpu *cpus = napid->napi_cpu; 1160 1161 napid->lilcl_head = napid->bigcl_head = -1; 1162 1163 do { 1164 more = 0; 1165 it++; curcl = -1; 1166 for (i = 0; i < NR_CPUS; i++) { 1167 cl = cpus[i].cluster_id; 1168 NAPI_DEBUG("Processing cpu[%d], cluster=%d\n", 1169 i, cl); 1170 if ((cl < HNC_MIN_CLUSTER) || (cl > HNC_MAX_CLUSTER)) { 1171 NAPI_DEBUG("Bad cluster (%d). SKIPPED\n", cl); 1172 /* continue if ASSERTs are disabled */ 1173 continue; 1174 }; 1175 if (cpumask_weight(&(cpus[i].core_mask)) == 0) { 1176 NAPI_DEBUG("Core mask 0. SKIPPED\n"); 1177 continue; 1178 } 1179 if (cl_done & (0x01 << cl)) { 1180 NAPI_DEBUG("Cluster already processed. SKIPPED\n"); 1181 continue; 1182 } else { 1183 if (more == 0) { 1184 more = 1; 1185 curcl = cl; 1186 curclhead = i; /* row */ 1187 clfrq = cpus[i].max_freq; 1188 prev = -1; 1189 }; 1190 if ((curcl >= 0) && (curcl != cl)) { 1191 NAPI_DEBUG("Entry cl(%d) != curcl(%d). SKIPPED\n", 1192 cl, curcl); 1193 continue; 1194 } 1195 if (cpus[i].max_freq != clfrq) 1196 NAPI_DEBUG("WARN: frq(%d)!=clfrq(%d)\n", 1197 cpus[i].max_freq, clfrq); 1198 if (clfrq >= bigfrq) { 1199 bigfrq = clfrq; 1200 napid->bigcl_head = curclhead; 1201 NAPI_DEBUG("bigcl=%d\n", curclhead); 1202 } 1203 if (clfrq < lilfrq) { 1204 lilfrq = clfrq; 1205 napid->lilcl_head = curclhead; 1206 NAPI_DEBUG("lilcl=%d\n", curclhead); 1207 } 1208 if (prev != -1) 1209 cpus[prev].cluster_nxt = i; 1210 1211 prev = i; 1212 } 1213 } 1214 if (curcl >= 0) 1215 cl_done |= (0x01 << curcl); 1216 1217 } while (more); 1218 1219 if (qdf_unlikely((napid->lilcl_head < 0) && (napid->bigcl_head < 0))) 1220 rc = -EFAULT; 1221 1222 hnc_dump_cpus(napid); /* if NAPI_DEBUG */ 1223 return rc; 1224 } 1225 #undef HNC_MIN_CLUSTER 1226 #undef HNC_MAX_CLUSTER 1227 1228 /* 1229 * hotplug function group 1230 */ 1231 1232 /** 1233 * hnc_cpu_online_cb() - handles CPU hotplug "up" events 1234 * @context: the associated HIF context 1235 * @cpu: the CPU Id of the CPU the event happened on 1236 * 1237 * Return: None 1238 */ 1239 static void hnc_cpu_online_cb(void *context, uint32_t cpu) 1240 { 1241 struct hif_softc *hif = context; 1242 struct qca_napi_data *napid = &hif->napi_data; 1243 1244 if (cpu >= NR_CPUS) 1245 return; 1246 1247 NAPI_DEBUG("-->%s(act=online, cpu=%u)", __func__, cpu); 1248 1249 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_UP; 1250 NAPI_DEBUG("%s: CPU %u marked %d", 1251 __func__, cpu, napid->napi_cpu[cpu].state); 1252 1253 NAPI_DEBUG("<--%s", __func__); 1254 } 1255 1256 /** 1257 * hnc_cpu_before_offline_cb() - handles CPU hotplug "prepare down" events 1258 * @context: the associated HIF context 1259 * @cpu: the CPU Id of the CPU the event happened on 1260 * 1261 * On transtion to offline, we act on PREP events, because we may need to move 1262 * the irqs/NAPIs to another CPU before it is actually off-lined. 1263 * 1264 * Return: None 1265 */ 1266 static void hnc_cpu_before_offline_cb(void *context, uint32_t cpu) 1267 { 1268 struct hif_softc *hif = context; 1269 struct qca_napi_data *napid = &hif->napi_data; 1270 1271 if (cpu >= NR_CPUS) 1272 return; 1273 1274 NAPI_DEBUG("-->%s(act=before_offline, cpu=%u)", __func__, cpu); 1275 1276 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_DOWN; 1277 1278 NAPI_DEBUG("%s: CPU %u marked %d; updating affinity", 1279 __func__, cpu, napid->napi_cpu[cpu].state); 1280 1281 /** 1282 * we need to move any NAPIs on this CPU out. 1283 * if we are in LO throughput mode, then this is valid 1284 * if the CPU is the the low designated CPU. 1285 */ 1286 hif_napi_event(GET_HIF_OPAQUE_HDL(hif), 1287 NAPI_EVT_CPU_STATE, 1288 (void *) 1289 ((size_t)cpu << 16 | napid->napi_cpu[cpu].state)); 1290 1291 NAPI_DEBUG("<--%s", __func__); 1292 } 1293 1294 static int hnc_hotplug_register(struct hif_softc *hif_sc) 1295 { 1296 QDF_STATUS status; 1297 1298 NAPI_DEBUG("-->%s", __func__); 1299 1300 status = qdf_cpuhp_register(&hif_sc->napi_data.cpuhp_handler, 1301 hif_sc, 1302 hnc_cpu_online_cb, 1303 hnc_cpu_before_offline_cb); 1304 1305 NAPI_DEBUG("<--%s [%d]", __func__, status); 1306 1307 return qdf_status_to_os_return(status); 1308 } 1309 1310 static void hnc_hotplug_unregister(struct hif_softc *hif_sc) 1311 { 1312 NAPI_DEBUG("-->%s", __func__); 1313 1314 if (hif_sc->napi_data.cpuhp_handler) 1315 qdf_cpuhp_unregister(&hif_sc->napi_data.cpuhp_handler); 1316 1317 NAPI_DEBUG("<--%s", __func__); 1318 } 1319 1320 /** 1321 * hnc_install_tput() - installs a callback in the throughput detector 1322 * @register: !0 => register; =0: unregister 1323 * 1324 * installs a callback to be called when wifi driver throughput (tx+rx) 1325 * crosses a threshold. Currently, we are using the same criteria as 1326 * TCP ack suppression (500 packets/100ms by default). 1327 * 1328 * Return: 0 : success 1329 * <0: failure 1330 */ 1331 1332 static int hnc_tput_hook(int install) 1333 { 1334 int rc = 0; 1335 1336 /* 1337 * Nothing, until the bw_calculation accepts registration 1338 * it is now hardcoded in the wlan_hdd_main.c::hdd_bus_bw_compute_cbk 1339 * hdd_napi_throughput_policy(...) 1340 */ 1341 return rc; 1342 } 1343 1344 /* 1345 * Implementation of hif_napi_cpu API 1346 */ 1347 1348 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) 1349 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1350 { 1351 cpumask_copy(&(cpus[i].thread_mask), 1352 topology_sibling_cpumask(i)); 1353 } 1354 #else 1355 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1356 { 1357 } 1358 #endif 1359 1360 1361 /** 1362 * hif_napi_cpu_init() - initialization of irq affinity block 1363 * @ctx: pointer to qca_napi_data 1364 * 1365 * called by hif_napi_create, after the first instance is called 1366 * - builds napi_rss_cpus table from cpu topology 1367 * - links cores of the same clusters together 1368 * - installs hot-plug notifier 1369 * - installs throughput trigger notifier (when such mechanism exists) 1370 * 1371 * Return: 0: OK 1372 * <0: error code 1373 */ 1374 int hif_napi_cpu_init(struct hif_opaque_softc *hif) 1375 { 1376 int rc = 0; 1377 int i; 1378 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1379 struct qca_napi_cpu *cpus = napid->napi_cpu; 1380 1381 NAPI_DEBUG("--> "); 1382 1383 if (cpus[0].state != QCA_NAPI_CPU_UNINITIALIZED) { 1384 NAPI_DEBUG("NAPI RSS table already initialized.\n"); 1385 rc = -EALREADY; 1386 goto lab_rss_init; 1387 } 1388 1389 /* build CPU topology table */ 1390 for_each_possible_cpu(i) { 1391 cpus[i].state = ((cpumask_test_cpu(i, cpu_online_mask) 1392 ? QCA_NAPI_CPU_UP 1393 : QCA_NAPI_CPU_DOWN)); 1394 cpus[i].core_id = topology_core_id(i); 1395 cpus[i].cluster_id = topology_physical_package_id(i); 1396 cpumask_copy(&(cpus[i].core_mask), 1397 topology_core_cpumask(i)); 1398 record_sibling_cpumask(cpus, i); 1399 cpus[i].max_freq = cpufreq_quick_get_max(i); 1400 cpus[i].napis = 0x0; 1401 cpus[i].cluster_nxt = -1; /* invalid */ 1402 } 1403 1404 /* link clusters together */ 1405 rc = hnc_link_clusters(napid); 1406 if (0 != rc) 1407 goto lab_err_topology; 1408 1409 /* install hotplug notifier */ 1410 rc = hnc_hotplug_register(HIF_GET_SOFTC(hif)); 1411 if (0 != rc) 1412 goto lab_err_hotplug; 1413 1414 /* install throughput notifier */ 1415 rc = hnc_tput_hook(1); 1416 if (0 == rc) 1417 goto lab_rss_init; 1418 1419 lab_err_hotplug: 1420 hnc_tput_hook(0); 1421 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1422 lab_err_topology: 1423 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1424 lab_rss_init: 1425 NAPI_DEBUG("<-- [rc=%d]", rc); 1426 return rc; 1427 } 1428 1429 /** 1430 * hif_napi_cpu_deinit() - clean-up of irq affinity block 1431 * 1432 * called by hif_napi_destroy, when the last instance is removed 1433 * - uninstalls throughput and hotplug notifiers 1434 * - clears cpu topology table 1435 * Return: 0: OK 1436 */ 1437 int hif_napi_cpu_deinit(struct hif_opaque_softc *hif) 1438 { 1439 int rc = 0; 1440 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1441 1442 NAPI_DEBUG("-->%s(...)", __func__); 1443 1444 /* uninstall tput notifier */ 1445 rc = hnc_tput_hook(0); 1446 1447 /* uninstall hotplug notifier */ 1448 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1449 1450 /* clear the topology table */ 1451 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1452 1453 NAPI_DEBUG("<--%s[rc=%d]", __func__, rc); 1454 1455 return rc; 1456 } 1457 1458 /** 1459 * hncm_migrate_to() - migrates a NAPI to a CPU 1460 * @napid: pointer to NAPI block 1461 * @ce_id: CE_id of the NAPI instance 1462 * @didx : index in the CPU topology table for the CPU to migrate to 1463 * 1464 * Migrates NAPI (identified by the CE_id) to the destination core 1465 * Updates the napi_map of the destination entry 1466 * 1467 * Return: 1468 * =0 : success 1469 * <0 : error 1470 */ 1471 static int hncm_migrate_to(struct qca_napi_data *napid, 1472 int napi_ce, 1473 int didx) 1474 { 1475 int rc = 0; 1476 QDF_STATUS status; 1477 1478 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, napi_ce, didx); 1479 1480 if (!napid->napis[napi_ce]) 1481 return -EINVAL; 1482 1483 napid->napis[napi_ce]->cpumask.bits[0] = (1 << didx); 1484 1485 irq_modify_status(napid->napis[napi_ce]->irq, IRQ_NO_BALANCING, 0); 1486 status = qdf_dev_set_irq_affinity(napid->napis[napi_ce]->irq, 1487 (struct qdf_cpu_mask *) 1488 &napid->napis[napi_ce]->cpumask); 1489 rc = qdf_status_to_os_return(status); 1490 1491 /* unmark the napis bitmap in the cpu table */ 1492 napid->napi_cpu[napid->napis[napi_ce]->cpu].napis &= ~(0x01 << napi_ce); 1493 /* mark the napis bitmap for the new designated cpu */ 1494 napid->napi_cpu[didx].napis |= (0x01 << napi_ce); 1495 napid->napis[napi_ce]->cpu = didx; 1496 1497 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1498 return rc; 1499 } 1500 /** 1501 * hncm_dest_cpu() - finds a destination CPU for NAPI 1502 * @napid: pointer to NAPI block 1503 * @act : RELOCATE | COLLAPSE | DISPERSE 1504 * 1505 * Finds the designated destionation for the next IRQ. 1506 * RELOCATE: translated to either COLLAPSE or DISPERSE based 1507 * on napid->napi_mode (throughput state) 1508 * COLLAPSE: All have the same destination: the first online CPU in lilcl 1509 * DISPERSE: One of the CPU in bigcl, which has the smallest number of 1510 * NAPIs on it 1511 * 1512 * Return: >=0 : index in the cpu topology table 1513 * : < 0 : error 1514 */ 1515 static int hncm_dest_cpu(struct qca_napi_data *napid, int act) 1516 { 1517 int destidx = -1; 1518 int head, i; 1519 1520 NAPI_DEBUG("-->%s(act=%d)", __func__, act); 1521 if (act == HNC_ACT_RELOCATE) { 1522 if (napid->napi_mode == QCA_NAPI_TPUT_LO) 1523 act = HNC_ACT_COLLAPSE; 1524 else 1525 act = HNC_ACT_DISPERSE; 1526 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d", 1527 __func__, act); 1528 } 1529 if (act == HNC_ACT_COLLAPSE) { 1530 head = i = napid->lilcl_head; 1531 retry_collapse: 1532 while (i >= 0) { 1533 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) { 1534 destidx = i; 1535 break; 1536 } 1537 i = napid->napi_cpu[i].cluster_nxt; 1538 } 1539 if ((destidx < 0) && (head == napid->lilcl_head)) { 1540 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl", 1541 __func__); 1542 head = i = napid->bigcl_head; 1543 goto retry_collapse; 1544 } 1545 } else { /* HNC_ACT_DISPERSE */ 1546 int smallest = 99; /* all 32 bits full */ 1547 int smallidx = -1; 1548 1549 head = i = napid->bigcl_head; 1550 retry_disperse: 1551 while (i >= 0) { 1552 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) && 1553 (hweight32(napid->napi_cpu[i].napis) <= smallest)) { 1554 smallest = napid->napi_cpu[i].napis; 1555 smallidx = i; 1556 } 1557 i = napid->napi_cpu[i].cluster_nxt; 1558 } 1559 /* Check if matches with user sepecified CPU mask */ 1560 smallidx = ((1 << smallidx) & napid->user_cpu_affin_mask) ? 1561 smallidx : -1; 1562 1563 if ((smallidx < 0) && (head == napid->bigcl_head)) { 1564 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl", 1565 __func__); 1566 head = i = napid->lilcl_head; 1567 goto retry_disperse; 1568 } 1569 destidx = smallidx; 1570 } 1571 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx); 1572 return destidx; 1573 } 1574 /** 1575 * hif_napi_cpu_migrate() - migrate IRQs away 1576 * @cpu: -1: all CPUs <n> specific CPU 1577 * @act: COLLAPSE | DISPERSE 1578 * 1579 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible 1580 * cores. Eligible cores are: 1581 * act=COLLAPSE -> the first online core of the little cluster 1582 * act=DISPERSE -> separate cores of the big cluster, so that each core will 1583 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis) 1584 * 1585 * Note that this function is called with a spinlock acquired already. 1586 * 1587 * Return: =0: success 1588 * <0: error 1589 */ 1590 1591 int hif_napi_cpu_migrate(struct qca_napi_data *napid, int cpu, int action) 1592 { 1593 int rc = 0; 1594 struct qca_napi_cpu *cpup; 1595 int i, dind; 1596 uint32_t napis; 1597 1598 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)", 1599 __func__, cpu, action); 1600 /* the following is really: hif_napi_enabled() with less overhead */ 1601 if (napid->ce_map == 0) { 1602 NAPI_DEBUG("%s: NAPI disabled. Not migrating.", __func__); 1603 goto hncm_return; 1604 } 1605 1606 cpup = napid->napi_cpu; 1607 1608 switch (action) { 1609 case HNC_ACT_RELOCATE: 1610 case HNC_ACT_DISPERSE: 1611 case HNC_ACT_COLLAPSE: { 1612 /* first find the src napi set */ 1613 if (cpu == HNC_ANY_CPU) 1614 napis = napid->ce_map; 1615 else 1616 napis = cpup[cpu].napis; 1617 /* then clear the napi bitmap on each CPU */ 1618 for (i = 0; i < NR_CPUS; i++) 1619 cpup[i].napis = 0; 1620 /* then for each of the NAPIs to disperse: */ 1621 for (i = 0; i < CE_COUNT_MAX; i++) 1622 if (napis & (1 << i)) { 1623 /* find a destination CPU */ 1624 dind = hncm_dest_cpu(napid, action); 1625 if (dind >= 0) { 1626 NAPI_DEBUG("Migrating NAPI ce%d to %d", 1627 i, dind); 1628 rc = hncm_migrate_to(napid, i, dind); 1629 } else { 1630 NAPI_DEBUG("No dest for NAPI ce%d", i); 1631 hnc_dump_cpus(napid); 1632 rc = -1; 1633 } 1634 } 1635 break; 1636 } 1637 default: { 1638 NAPI_DEBUG("%s: bad action: %d\n", __func__, action); 1639 QDF_BUG(0); 1640 break; 1641 } 1642 } /* switch action */ 1643 1644 hncm_return: 1645 hnc_dump_cpus(napid); 1646 return rc; 1647 } 1648 1649 1650 /** 1651 * hif_napi_bl_irq() - calls irq_modify_status to enable/disable blacklisting 1652 * @napid: pointer to qca_napi_data structure 1653 * @bl_flag: blacklist flag to enable/disable blacklisting 1654 * 1655 * The function enables/disables blacklisting for all the copy engine 1656 * interrupts on which NAPI is enabled. 1657 * 1658 * Return: None 1659 */ 1660 static inline void hif_napi_bl_irq(struct qca_napi_data *napid, bool bl_flag) 1661 { 1662 int i; 1663 struct qca_napi_info *napii; 1664 1665 for (i = 0; i < CE_COUNT_MAX; i++) { 1666 /* check if NAPI is enabled on the CE */ 1667 if (!(napid->ce_map & (0x01 << i))) 1668 continue; 1669 1670 /*double check that NAPI is allocated for the CE */ 1671 napii = napid->napis[i]; 1672 if (!(napii)) 1673 continue; 1674 1675 if (bl_flag == true) 1676 irq_modify_status(napii->irq, 1677 0, IRQ_NO_BALANCING); 1678 else 1679 irq_modify_status(napii->irq, 1680 IRQ_NO_BALANCING, 0); 1681 HIF_DBG("%s: bl_flag %d CE %d", __func__, bl_flag, i); 1682 } 1683 } 1684 1685 #ifdef CONFIG_SCHED_CORE_CTL 1686 /* Enable this API only if kernel feature - CONFIG_SCHED_CORE_CTL is defined */ 1687 static inline int hif_napi_core_ctl_set_boost(bool boost) 1688 { 1689 return core_ctl_set_boost(boost); 1690 } 1691 #else 1692 static inline int hif_napi_core_ctl_set_boost(bool boost) 1693 { 1694 return 0; 1695 } 1696 #endif 1697 /** 1698 * hif_napi_cpu_blacklist() - en(dis)ables blacklisting for NAPI RX interrupts. 1699 * @napid: pointer to qca_napi_data structure 1700 * @op: blacklist operation to perform 1701 * 1702 * The function enables/disables/queries blacklisting for all CE RX 1703 * interrupts with NAPI enabled. Besides blacklisting, it also enables/disables 1704 * core_ctl_set_boost. 1705 * Once blacklisting is enabled, the interrupts will not be managed by the IRQ 1706 * balancer. 1707 * 1708 * Return: -EINVAL, in case IRQ_BLACKLISTING and CORE_CTL_BOOST is not enabled 1709 * for BLACKLIST_QUERY op - blacklist refcount 1710 * for BLACKLIST_ON op - return value from core_ctl_set_boost API 1711 * for BLACKLIST_OFF op - return value from core_ctl_set_boost API 1712 */ 1713 int hif_napi_cpu_blacklist(struct qca_napi_data *napid, 1714 enum qca_blacklist_op op) 1715 { 1716 int rc = 0; 1717 static int ref_count; /* = 0 by the compiler */ 1718 uint8_t flags = napid->flags; 1719 bool bl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING; 1720 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST; 1721 1722 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op); 1723 1724 if (!(bl_en && ccb_en)) { 1725 rc = -EINVAL; 1726 goto out; 1727 } 1728 1729 switch (op) { 1730 case BLACKLIST_QUERY: 1731 rc = ref_count; 1732 break; 1733 case BLACKLIST_ON: 1734 ref_count++; 1735 rc = 0; 1736 if (ref_count == 1) { 1737 rc = hif_napi_core_ctl_set_boost(true); 1738 NAPI_DEBUG("boost_on() returns %d - refcnt=%d", 1739 rc, ref_count); 1740 hif_napi_bl_irq(napid, true); 1741 } 1742 break; 1743 case BLACKLIST_OFF: 1744 if (ref_count) { 1745 ref_count--; 1746 rc = 0; 1747 if (ref_count == 0) { 1748 rc = hif_napi_core_ctl_set_boost(false); 1749 NAPI_DEBUG("boost_off() returns %d - refcnt=%d", 1750 rc, ref_count); 1751 hif_napi_bl_irq(napid, false); 1752 } 1753 } 1754 break; 1755 default: 1756 NAPI_DEBUG("Invalid blacklist op: %d", op); 1757 rc = -EINVAL; 1758 } /* switch */ 1759 out: 1760 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1761 return rc; 1762 } 1763 1764 /** 1765 * hif_napi_serialize() - [de-]serialize NAPI operations 1766 * @hif: context 1767 * @is_on: 1: serialize, 0: deserialize 1768 * 1769 * hif_napi_serialize(hif, 1) can be called multiple times. It will perform the 1770 * following steps (see hif_napi_event for code): 1771 * - put irqs of all NAPI instances on the same CPU 1772 * - only for the first serialize call: blacklist 1773 * 1774 * hif_napi_serialize(hif, 0): 1775 * - start a timer (multiple of BusBandwidthTimer -- default: 100 msec) 1776 * - at the end of the timer, check the current throughput state and 1777 * implement it. 1778 */ 1779 static unsigned long napi_serialize_reqs; 1780 int hif_napi_serialize(struct hif_opaque_softc *hif, int is_on) 1781 { 1782 int rc = -EINVAL; 1783 1784 if (hif != NULL) 1785 switch (is_on) { 1786 case 0: { /* de-serialize */ 1787 rc = hif_napi_event(hif, NAPI_EVT_USR_NORMAL, 1788 (void *) 0); 1789 napi_serialize_reqs = 0; 1790 break; 1791 } /* end de-serialize */ 1792 case 1: { /* serialize */ 1793 rc = hif_napi_event(hif, NAPI_EVT_USR_SERIAL, 1794 (void *)napi_serialize_reqs++); 1795 break; 1796 } /* end serialize */ 1797 default: 1798 break; /* no-op */ 1799 } /* switch */ 1800 return rc; 1801 } 1802 1803 #endif /* ifdef HIF_IRQ_AFFINITY */ 1804