1 /* 2 * Copyright (c) 2015-2018 The Linux Foundation. All rights reserved. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for 5 * any purpose with or without fee is hereby granted, provided that the 6 * above copyright notice and this permission notice appear in all 7 * copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 10 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 11 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 12 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 13 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 14 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 15 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 * PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /** 20 * DOC: hif_napi.c 21 * 22 * HIF NAPI interface implementation 23 */ 24 25 #include <linux/string.h> /* memset */ 26 27 /* Linux headers */ 28 #include <linux/cpumask.h> 29 #include <linux/cpufreq.h> 30 #include <linux/cpu.h> 31 #include <linux/topology.h> 32 #include <linux/interrupt.h> 33 #include <linux/irq.h> 34 #ifdef CONFIG_SCHED_CORE_CTL 35 #include <linux/sched/core_ctl.h> 36 #endif 37 #include <pld_common.h> 38 #include <linux/pm.h> 39 40 /* Driver headers */ 41 #include <hif_napi.h> 42 #include <hif_debug.h> 43 #include <hif_io32.h> 44 #include <ce_api.h> 45 #include <ce_internal.h> 46 #include <hif_irq_affinity.h> 47 #include "qdf_cpuhp.h" 48 #include "qdf_module.h" 49 50 enum napi_decision_vector { 51 HIF_NAPI_NOEVENT = 0, 52 HIF_NAPI_INITED = 1, 53 HIF_NAPI_CONF_UP = 2 54 }; 55 #define ENABLE_NAPI_MASK (HIF_NAPI_INITED | HIF_NAPI_CONF_UP) 56 57 #ifdef RECEIVE_OFFLOAD 58 /** 59 * hif_rxthread_napi_poll() - dummy napi poll for rx_thread NAPI 60 * @napi: Rx_thread NAPI 61 * @budget: NAPI BUDGET 62 * 63 * Return: 0 as it is not supposed to be polled at all as it is not scheduled. 64 */ 65 static int hif_rxthread_napi_poll(struct napi_struct *napi, int budget) 66 { 67 HIF_ERROR("This napi_poll should not be polled as we don't schedule it"); 68 QDF_ASSERT(0); 69 return 0; 70 } 71 72 /** 73 * hif_init_rx_thread_napi() - Initialize dummy Rx_thread NAPI 74 * @napii: Handle to napi_info holding rx_thread napi 75 * 76 * Return: None 77 */ 78 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 79 { 80 init_dummy_netdev(&napii->rx_thread_netdev); 81 netif_napi_add(&napii->rx_thread_netdev, &napii->rx_thread_napi, 82 hif_rxthread_napi_poll, 64); 83 napi_enable(&napii->rx_thread_napi); 84 } 85 #else /* RECEIVE_OFFLOAD */ 86 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 87 { 88 } 89 #endif 90 91 /** 92 * hif_napi_create() - creates the NAPI structures for a given CE 93 * @hif : pointer to hif context 94 * @pipe_id: the CE id on which the instance will be created 95 * @poll : poll function to be used for this NAPI instance 96 * @budget : budget to be registered with the NAPI instance 97 * @scale : scale factor on the weight (to scaler budget to 1000) 98 * @flags : feature flags 99 * 100 * Description: 101 * Creates NAPI instances. This function is called 102 * unconditionally during initialization. It creates 103 * napi structures through the proper HTC/HIF calls. 104 * The structures are disabled on creation. 105 * Note that for each NAPI instance a separate dummy netdev is used 106 * 107 * Return: 108 * < 0: error 109 * = 0: <should never happen> 110 * > 0: id of the created object (for multi-NAPI, number of objects created) 111 */ 112 int hif_napi_create(struct hif_opaque_softc *hif_ctx, 113 int (*poll)(struct napi_struct *, int), 114 int budget, 115 int scale, 116 uint8_t flags) 117 { 118 int i; 119 struct qca_napi_data *napid; 120 struct qca_napi_info *napii; 121 struct CE_state *ce_state; 122 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 123 int rc = 0; 124 125 NAPI_DEBUG("-->(budget=%d, scale=%d)", 126 budget, scale); 127 NAPI_DEBUG("hif->napi_data.state = 0x%08x", 128 hif->napi_data.state); 129 NAPI_DEBUG("hif->napi_data.ce_map = 0x%08x", 130 hif->napi_data.ce_map); 131 132 napid = &(hif->napi_data); 133 if (0 == (napid->state & HIF_NAPI_INITED)) { 134 memset(napid, 0, sizeof(struct qca_napi_data)); 135 qdf_spinlock_create(&(napid->lock)); 136 137 napid->state |= HIF_NAPI_INITED; 138 napid->flags = flags; 139 140 rc = hif_napi_cpu_init(hif_ctx); 141 if (rc != 0 && rc != -EALREADY) { 142 HIF_ERROR("NAPI_initialization failed,. %d", rc); 143 rc = napid->ce_map; 144 goto hnc_err; 145 } else 146 rc = 0; 147 148 HIF_DBG("%s: NAPI structures initialized, rc=%d", 149 __func__, rc); 150 } 151 for (i = 0; i < hif->ce_count; i++) { 152 ce_state = hif->ce_id_to_state[i]; 153 NAPI_DEBUG("ce %d: htt_rx=%d htt_tx=%d", 154 i, ce_state->htt_rx_data, 155 ce_state->htt_tx_data); 156 if (ce_srng_based(hif)) 157 continue; 158 159 if (!ce_state->htt_rx_data) 160 continue; 161 162 /* Now this is a CE where we need NAPI on */ 163 NAPI_DEBUG("Creating NAPI on pipe %d", i); 164 napii = qdf_mem_malloc(sizeof(*napii)); 165 napid->napis[i] = napii; 166 if (!napii) { 167 NAPI_DEBUG("NAPI alloc failure %d", i); 168 rc = -ENOMEM; 169 goto napii_alloc_failure; 170 } 171 } 172 173 for (i = 0; i < hif->ce_count; i++) { 174 napii = napid->napis[i]; 175 if (!napii) 176 continue; 177 178 NAPI_DEBUG("initializing NAPI for pipe %d", i); 179 memset(napii, 0, sizeof(struct qca_napi_info)); 180 napii->scale = scale; 181 napii->id = NAPI_PIPE2ID(i); 182 napii->hif_ctx = hif_ctx; 183 napii->irq = pld_get_irq(hif->qdf_dev->dev, i); 184 185 if (napii->irq < 0) 186 HIF_WARN("%s: bad IRQ value for CE %d: %d", 187 __func__, i, napii->irq); 188 189 init_dummy_netdev(&(napii->netdev)); 190 191 NAPI_DEBUG("adding napi=%pK to netdev=%pK (poll=%pK, bdgt=%d)", 192 &(napii->napi), &(napii->netdev), poll, budget); 193 netif_napi_add(&(napii->netdev), &(napii->napi), poll, budget); 194 195 NAPI_DEBUG("after napi_add"); 196 NAPI_DEBUG("napi=0x%pK, netdev=0x%pK", 197 &(napii->napi), &(napii->netdev)); 198 NAPI_DEBUG("napi.dev_list.prev=0x%pK, next=0x%pK", 199 napii->napi.dev_list.prev, 200 napii->napi.dev_list.next); 201 NAPI_DEBUG("dev.napi_list.prev=0x%pK, next=0x%pK", 202 napii->netdev.napi_list.prev, 203 napii->netdev.napi_list.next); 204 205 hif_init_rx_thread_napi(napii); 206 napii->lro_ctx = qdf_lro_init(); 207 NAPI_DEBUG("Registering LRO for ce_id %d NAPI callback for %d lro_ctx %pK\n", 208 i, napii->id, napii->lro_ctx); 209 210 /* It is OK to change the state variable below without 211 * protection as there should be no-one around yet 212 */ 213 napid->ce_map |= (0x01 << i); 214 HIF_DBG("%s: NAPI id %d created for pipe %d", __func__, 215 napii->id, i); 216 } 217 218 /* no ces registered with the napi */ 219 if (!ce_srng_based(hif) && napid->ce_map == 0) { 220 HIF_WARN("%s: no napis created for copy engines", __func__); 221 return -EFAULT; 222 } 223 224 NAPI_DEBUG("napi map = %x", napid->ce_map); 225 NAPI_DEBUG("NAPI ids created for all applicable pipes"); 226 return napid->ce_map; 227 228 napii_alloc_failure: 229 for (i = 0; i < hif->ce_count; i++) { 230 napii = napid->napis[i]; 231 napid->napis[i] = NULL; 232 if (napii) 233 qdf_mem_free(napii); 234 } 235 236 hnc_err: 237 NAPI_DEBUG("<--napi_instances_map=%x]", napid->ce_map); 238 return rc; 239 } 240 qdf_export_symbol(hif_napi_create); 241 242 #ifdef RECEIVE_OFFLOAD 243 void hif_napi_rx_offld_flush_cb_register(struct hif_opaque_softc *hif_hdl, 244 void (offld_flush_handler)(void *)) 245 { 246 int i; 247 struct CE_state *ce_state; 248 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 249 struct qca_napi_data *napid; 250 struct qca_napi_info *napii; 251 252 if (!scn) { 253 HIF_ERROR("%s: hif_state NULL!", __func__); 254 QDF_ASSERT(0); 255 return; 256 } 257 258 napid = hif_napi_get_all(hif_hdl); 259 for (i = 0; i < scn->ce_count; i++) { 260 ce_state = scn->ce_id_to_state[i]; 261 if (ce_state && (ce_state->htt_rx_data)) { 262 napii = napid->napis[i]; 263 napii->offld_flush_cb = offld_flush_handler; 264 HIF_DBG("Registering offload for ce_id %d NAPI callback for %d flush_cb %p\n", 265 i, napii->id, napii->offld_flush_cb); 266 } 267 } 268 } 269 270 void hif_napi_rx_offld_flush_cb_deregister(struct hif_opaque_softc *hif_hdl) 271 { 272 int i; 273 struct CE_state *ce_state; 274 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 275 struct qca_napi_data *napid; 276 struct qca_napi_info *napii; 277 278 if (!scn) { 279 HIF_ERROR("%s: hif_state NULL!", __func__); 280 QDF_ASSERT(0); 281 return; 282 } 283 284 napid = hif_napi_get_all(hif_hdl); 285 for (i = 0; i < scn->ce_count; i++) { 286 ce_state = scn->ce_id_to_state[i]; 287 if (ce_state && (ce_state->htt_rx_data)) { 288 napii = napid->napis[i]; 289 HIF_DBG("deRegistering offld for ce_id %d NAPI callback for %d flush_cb %pK\n", 290 i, napii->id, napii->offld_flush_cb); 291 /* Not required */ 292 napii->offld_flush_cb = NULL; 293 } 294 } 295 } 296 #endif /* RECEIVE_OFFLOAD */ 297 298 /** 299 * 300 * hif_napi_destroy() - destroys the NAPI structures for a given instance 301 * @hif : pointer to hif context 302 * @ce_id : the CE id whose napi instance will be destroyed 303 * @force : if set, will destroy even if entry is active (de-activates) 304 * 305 * Description: 306 * Destroy a given NAPI instance. This function is called 307 * unconditionally during cleanup. 308 * Refuses to destroy an entry of it is still enabled (unless force=1) 309 * Marks the whole napi_data invalid if all instances are destroyed. 310 * 311 * Return: 312 * -EINVAL: specific entry has not been created 313 * -EPERM : specific entry is still active 314 * 0 < : error 315 * 0 = : success 316 */ 317 int hif_napi_destroy(struct hif_opaque_softc *hif_ctx, 318 uint8_t id, 319 int force) 320 { 321 uint8_t ce = NAPI_ID2PIPE(id); 322 int rc = 0; 323 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 324 325 NAPI_DEBUG("-->(id=%d, force=%d)", id, force); 326 327 if (0 == (hif->napi_data.state & HIF_NAPI_INITED)) { 328 HIF_ERROR("%s: NAPI not initialized or entry %d not created", 329 __func__, id); 330 rc = -EINVAL; 331 } else if (0 == (hif->napi_data.ce_map & (0x01 << ce))) { 332 HIF_ERROR("%s: NAPI instance %d (pipe %d) not created", 333 __func__, id, ce); 334 if (hif->napi_data.napis[ce]) 335 HIF_ERROR("%s: memory allocated but ce_map not set %d (pipe %d)", 336 __func__, id, ce); 337 rc = -EINVAL; 338 } else { 339 struct qca_napi_data *napid; 340 struct qca_napi_info *napii; 341 342 napid = &(hif->napi_data); 343 napii = napid->napis[ce]; 344 if (!napii) { 345 if (napid->ce_map & (0x01 << ce)) 346 HIF_ERROR("%s: napii & ce_map out of sync(ce %d)", 347 __func__, ce); 348 return -EINVAL; 349 } 350 351 352 if (hif->napi_data.state == HIF_NAPI_CONF_UP) { 353 if (force) { 354 napi_disable(&(napii->napi)); 355 HIF_DBG("%s: NAPI entry %d force disabled", 356 __func__, id); 357 NAPI_DEBUG("NAPI %d force disabled", id); 358 } else { 359 HIF_ERROR("%s: Cannot destroy active NAPI %d", 360 __func__, id); 361 rc = -EPERM; 362 } 363 } 364 if (0 == rc) { 365 NAPI_DEBUG("before napi_del"); 366 NAPI_DEBUG("napi.dlist.prv=0x%pK, next=0x%pK", 367 napii->napi.dev_list.prev, 368 napii->napi.dev_list.next); 369 NAPI_DEBUG("dev.napi_l.prv=0x%pK, next=0x%pK", 370 napii->netdev.napi_list.prev, 371 napii->netdev.napi_list.next); 372 373 qdf_lro_deinit(napii->lro_ctx); 374 netif_napi_del(&(napii->napi)); 375 376 napid->ce_map &= ~(0x01 << ce); 377 napid->napis[ce] = NULL; 378 napii->scale = 0; 379 qdf_mem_free(napii); 380 HIF_DBG("%s: NAPI %d destroyed\n", __func__, id); 381 382 /* if there are no active instances and 383 * if they are all destroyed, 384 * set the whole structure to uninitialized state 385 */ 386 if (napid->ce_map == 0) { 387 rc = hif_napi_cpu_deinit(hif_ctx); 388 /* caller is tolerant to receiving !=0 rc */ 389 390 qdf_spinlock_destroy(&(napid->lock)); 391 memset(napid, 392 0, sizeof(struct qca_napi_data)); 393 HIF_DBG("%s: no NAPI instances. Zapped.", 394 __func__); 395 } 396 } 397 } 398 399 return rc; 400 } 401 qdf_export_symbol(hif_napi_destroy); 402 403 #ifdef FEATURE_LRO 404 void *hif_napi_get_lro_info(struct hif_opaque_softc *hif_hdl, int napi_id) 405 { 406 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 407 struct qca_napi_data *napid; 408 struct qca_napi_info *napii; 409 410 napid = &(scn->napi_data); 411 napii = napid->napis[NAPI_ID2PIPE(napi_id)]; 412 413 if (napii) 414 return napii->lro_ctx; 415 return 0; 416 } 417 #endif 418 419 /** 420 * 421 * hif_napi_get_all() - returns the address of the whole HIF NAPI structure 422 * @hif: pointer to hif context 423 * 424 * Description: 425 * Returns the address of the whole structure 426 * 427 * Return: 428 * <addr>: address of the whole HIF NAPI structure 429 */ 430 inline struct qca_napi_data *hif_napi_get_all(struct hif_opaque_softc *hif_ctx) 431 { 432 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 433 434 return &(hif->napi_data); 435 } 436 437 struct qca_napi_info *hif_get_napi(int napi_id, struct qca_napi_data *napid) 438 { 439 int id = NAPI_ID2PIPE(napi_id); 440 441 return napid->napis[id]; 442 } 443 444 /** 445 * 446 * hif_napi_event() - reacts to events that impact NAPI 447 * @hif : pointer to hif context 448 * @evnt: event that has been detected 449 * @data: more data regarding the event 450 * 451 * Description: 452 * This function handles two types of events: 453 * 1- Events that change the state of NAPI (enabled/disabled): 454 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE} 455 * The state is retrievable by "hdd_napi_enabled(-1)" 456 * - NAPI will be on if either INI file is on and it has not been disabled 457 * by a subsequent vendor CMD, 458 * or it has been enabled by a vendor CMD. 459 * 2- Events that change the CPU affinity of a NAPI instance/IRQ: 460 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE} 461 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode 462 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy() 463 * - In LO tput mode, NAPI will yield control if its interrupts to the system 464 * management functions. However in HI throughput mode, NAPI will actively 465 * manage its interrupts/instances (by trying to disperse them out to 466 * separate performance cores). 467 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events. 468 * 469 * + In some cases (roaming peer management is the only case so far), a 470 * a client can trigger a "SERIALIZE" event. Basically, this means that the 471 * users is asking NAPI to go into a truly single execution context state. 472 * So, NAPI indicates to msm-irqbalancer that it wants to be blacklisted, 473 * (if called for the first time) and then moves all IRQs (for NAPI 474 * instances) to be collapsed to a single core. If called multiple times, 475 * it will just re-collapse the CPUs. This is because blacklist-on() API 476 * is reference-counted, and because the API has already been called. 477 * 478 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go 479 * to its "normal" operation. Optionally, they can give a timeout value (in 480 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this 481 * case, NAPI will just set the current throughput state to uninitialized 482 * and set the delay period. Once policy handler is called, it would skip 483 * applying the policy delay period times, and otherwise apply the policy. 484 * 485 * Return: 486 * < 0: some error 487 * = 0: event handled successfully 488 */ 489 int hif_napi_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event, 490 void *data) 491 { 492 int rc = 0; 493 uint32_t prev_state; 494 int i; 495 bool state_changed; 496 struct napi_struct *napi; 497 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 498 struct qca_napi_data *napid = &(hif->napi_data); 499 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED; 500 enum { 501 BLACKLIST_NOT_PENDING, 502 BLACKLIST_ON_PENDING, 503 BLACKLIST_OFF_PENDING 504 } blacklist_pending = BLACKLIST_NOT_PENDING; 505 506 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data); 507 508 if (ce_srng_based(hif)) 509 return hif_exec_event(hif_ctx, event, data); 510 511 if ((napid->state & HIF_NAPI_INITED) == 0) { 512 NAPI_DEBUG("%s: got event when NAPI not initialized", 513 __func__); 514 return -EINVAL; 515 } 516 qdf_spin_lock_bh(&(napid->lock)); 517 prev_state = napid->state; 518 switch (event) { 519 case NAPI_EVT_INI_FILE: 520 case NAPI_EVT_CMD_STATE: 521 case NAPI_EVT_INT_STATE: { 522 int on = (data != ((void *)0)); 523 524 HIF_DBG("%s: recved evnt: STATE_CMD %d; v = %d (state=0x%0x)", 525 __func__, event, 526 on, prev_state); 527 if (on) 528 if (prev_state & HIF_NAPI_CONF_UP) { 529 HIF_DBG("%s: duplicate NAPI conf ON msg", 530 __func__); 531 } else { 532 HIF_DBG("%s: setting state to ON", 533 __func__); 534 napid->state |= HIF_NAPI_CONF_UP; 535 } 536 else /* off request */ 537 if (prev_state & HIF_NAPI_CONF_UP) { 538 HIF_DBG("%s: setting state to OFF", 539 __func__); 540 napid->state &= ~HIF_NAPI_CONF_UP; 541 } else { 542 HIF_DBG("%s: duplicate NAPI conf OFF msg", 543 __func__); 544 } 545 break; 546 } 547 /* case NAPI_INIT_FILE/CMD_STATE */ 548 549 case NAPI_EVT_CPU_STATE: { 550 int cpu = ((unsigned long int)data >> 16); 551 int val = ((unsigned long int)data & 0x0ff); 552 553 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d", 554 __func__, cpu, val); 555 556 /* state has already been set by hnc_cpu_notify_cb */ 557 if ((val == QCA_NAPI_CPU_DOWN) && 558 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */ 559 (napid->napi_cpu[cpu].napis != 0)) { 560 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d", 561 __func__, cpu); 562 rc = hif_napi_cpu_migrate(napid, 563 cpu, 564 HNC_ACT_RELOCATE); 565 napid->napi_cpu[cpu].napis = 0; 566 } 567 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */ 568 break; 569 } 570 571 case NAPI_EVT_TPUT_STATE: { 572 tput_mode = (enum qca_napi_tput_state)data; 573 if (tput_mode == QCA_NAPI_TPUT_LO) { 574 /* from TPUT_HI -> TPUT_LO */ 575 NAPI_DEBUG("%s: Moving to napi_tput_LO state", 576 __func__); 577 blacklist_pending = BLACKLIST_OFF_PENDING; 578 /* 579 * Ideally we should "collapse" interrupts here, since 580 * we are "dispersing" interrupts in the "else" case. 581 * This allows the possibility that our interrupts may 582 * still be on the perf cluster the next time we enter 583 * high tput mode. However, the irq_balancer is free 584 * to move our interrupts to power cluster once 585 * blacklisting has been turned off in the "else" case. 586 */ 587 } else { 588 /* from TPUT_LO -> TPUT->HI */ 589 NAPI_DEBUG("%s: Moving to napi_tput_HI state", 590 __func__); 591 rc = hif_napi_cpu_migrate(napid, 592 HNC_ANY_CPU, 593 HNC_ACT_DISPERSE); 594 595 blacklist_pending = BLACKLIST_ON_PENDING; 596 } 597 napid->napi_mode = tput_mode; 598 break; 599 } 600 601 case NAPI_EVT_USR_SERIAL: { 602 unsigned long users = (unsigned long)data; 603 604 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld", 605 __func__, users); 606 607 rc = hif_napi_cpu_migrate(napid, 608 HNC_ANY_CPU, 609 HNC_ACT_COLLAPSE); 610 if ((users == 0) && (rc == 0)) 611 blacklist_pending = BLACKLIST_ON_PENDING; 612 break; 613 } 614 case NAPI_EVT_USR_NORMAL: { 615 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__); 616 /* 617 * Deserialization timeout is handled at hdd layer; 618 * just mark current mode to uninitialized to ensure 619 * it will be set when the delay is over 620 */ 621 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED; 622 break; 623 } 624 default: { 625 HIF_ERROR("%s: unknown event: %d (data=0x%0lx)", 626 __func__, event, (unsigned long) data); 627 break; 628 } /* default */ 629 }; /* switch */ 630 631 632 switch (blacklist_pending) { 633 case BLACKLIST_ON_PENDING: 634 /* assume the control of WLAN IRQs */ 635 hif_napi_cpu_blacklist(napid, BLACKLIST_ON); 636 break; 637 case BLACKLIST_OFF_PENDING: 638 /* yield the control of WLAN IRQs */ 639 hif_napi_cpu_blacklist(napid, BLACKLIST_OFF); 640 break; 641 default: /* nothing to do */ 642 break; 643 } /* switch blacklist_pending */ 644 645 /* we want to perform the comparison in lock: 646 * there is a possiblity of hif_napi_event get called 647 * from two different contexts (driver unload and cpu hotplug 648 * notification) and napid->state get changed 649 * in driver unload context and can lead to race condition 650 * in cpu hotplug context. Therefore, perform the napid->state 651 * comparison before releasing lock. 652 */ 653 state_changed = (prev_state != napid->state); 654 qdf_spin_unlock_bh(&(napid->lock)); 655 656 if (state_changed) { 657 if (napid->state == ENABLE_NAPI_MASK) { 658 rc = 1; 659 for (i = 0; i < CE_COUNT_MAX; i++) { 660 struct qca_napi_info *napii = napid->napis[i]; 661 if (napii) { 662 napi = &(napii->napi); 663 NAPI_DEBUG("%s: enabling NAPI %d", 664 __func__, i); 665 napi_enable(napi); 666 } 667 } 668 } else { 669 rc = 0; 670 for (i = 0; i < CE_COUNT_MAX; i++) { 671 struct qca_napi_info *napii = napid->napis[i]; 672 if (napii) { 673 napi = &(napii->napi); 674 NAPI_DEBUG("%s: disabling NAPI %d", 675 __func__, i); 676 napi_disable(napi); 677 /* in case it is affined, remove it */ 678 irq_set_affinity_hint(napii->irq, NULL); 679 } 680 } 681 } 682 } else { 683 HIF_DBG("%s: no change in hif napi state (still %d)", 684 __func__, prev_state); 685 } 686 687 NAPI_DEBUG("<--[rc=%d]", rc); 688 return rc; 689 } 690 qdf_export_symbol(hif_napi_event); 691 692 /** 693 * hif_napi_enabled() - checks whether NAPI is enabled for given ce or not 694 * @hif: hif context 695 * @ce : CE instance (or -1, to check if any CEs are enabled) 696 * 697 * Return: bool 698 */ 699 int hif_napi_enabled(struct hif_opaque_softc *hif_ctx, int ce) 700 { 701 int rc; 702 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 703 704 if (-1 == ce) 705 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK)); 706 else 707 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK) && 708 (hif->napi_data.ce_map & (0x01 << ce))); 709 return rc; 710 } 711 qdf_export_symbol(hif_napi_enabled); 712 713 /** 714 * hif_napi_enable_irq() - enables bus interrupts after napi_complete 715 * 716 * @hif: hif context 717 * @id : id of NAPI instance calling this (used to determine the CE) 718 * 719 * Return: void 720 */ 721 inline void hif_napi_enable_irq(struct hif_opaque_softc *hif, int id) 722 { 723 struct hif_softc *scn = HIF_GET_SOFTC(hif); 724 725 hif_irq_enable(scn, NAPI_ID2PIPE(id)); 726 } 727 728 729 /** 730 * hif_napi_schedule() - schedules napi, updates stats 731 * @scn: hif context 732 * @ce_id: index of napi instance 733 * 734 * Return: void 735 */ 736 int hif_napi_schedule(struct hif_opaque_softc *hif_ctx, int ce_id) 737 { 738 int cpu = smp_processor_id(); 739 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 740 struct qca_napi_info *napii; 741 742 hif_record_ce_desc_event(scn, ce_id, NAPI_SCHEDULE, 743 NULL, NULL, 0, 0); 744 745 napii = scn->napi_data.napis[ce_id]; 746 if (qdf_unlikely(!napii)) { 747 HIF_ERROR("%s, scheduling unallocated napi (ce:%d)", 748 __func__, ce_id); 749 qdf_atomic_dec(&scn->active_tasklet_cnt); 750 return false; 751 } 752 753 napii->stats[cpu].napi_schedules++; 754 NAPI_DEBUG("scheduling napi %d (ce:%d)", napii->id, ce_id); 755 napi_schedule(&(napii->napi)); 756 757 return true; 758 } 759 qdf_export_symbol(hif_napi_schedule); 760 761 /** 762 * hif_napi_correct_cpu() - correct the interrupt affinity for napi if needed 763 * @napi_info: pointer to qca_napi_info for the napi instance 764 * 765 * Return: true => interrupt already on correct cpu, no correction needed 766 * false => interrupt on wrong cpu, correction done for cpu affinity 767 * of the interrupt 768 */ 769 static inline 770 bool hif_napi_correct_cpu(struct qca_napi_info *napi_info) 771 { 772 bool right_cpu = true; 773 int rc = 0; 774 cpumask_t cpumask; 775 int cpu; 776 struct qca_napi_data *napid; 777 778 napid = hif_napi_get_all(GET_HIF_OPAQUE_HDL(napi_info->hif_ctx)); 779 780 if (napid->flags & QCA_NAPI_FEATURE_CPU_CORRECTION) { 781 782 cpu = qdf_get_cpu(); 783 if (unlikely((hif_napi_cpu_blacklist(napid, 784 BLACKLIST_QUERY) > 0) && 785 (cpu != napi_info->cpu))) { 786 right_cpu = false; 787 788 NAPI_DEBUG("interrupt on wrong CPU, correcting"); 789 cpumask.bits[0] = (0x01 << napi_info->cpu); 790 791 irq_modify_status(napi_info->irq, IRQ_NO_BALANCING, 0); 792 rc = irq_set_affinity_hint(napi_info->irq, 793 &cpumask); 794 irq_modify_status(napi_info->irq, 0, IRQ_NO_BALANCING); 795 796 if (rc) 797 HIF_ERROR("error setting irq affinity hint: %d", 798 rc); 799 else 800 napi_info->stats[cpu].cpu_corrected++; 801 } 802 } 803 return right_cpu; 804 } 805 806 #ifdef RECEIVE_OFFLOAD 807 /** 808 * hif_napi_offld_flush_cb() - Call upper layer flush callback 809 * @napi_info: Handle to hif_napi_info 810 * 811 * Return: None 812 */ 813 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 814 { 815 if (napi_info->offld_flush_cb) 816 napi_info->offld_flush_cb(napi_info); 817 } 818 #else 819 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 820 { 821 } 822 #endif 823 824 /** 825 * hif_napi_poll() - NAPI poll routine 826 * @napi : pointer to NAPI struct as kernel holds it 827 * @budget: 828 * 829 * This is the body of the poll function. 830 * The poll function is called by kernel. So, there is a wrapper 831 * function in HDD, which in turn calls this function. 832 * Two main reasons why the whole thing is not implemented in HDD: 833 * a) references to things like ce_service that HDD is not aware of 834 * b) proximity to the implementation of ce_tasklet, which the body 835 * of this function should be very close to. 836 * 837 * NOTE TO THE MAINTAINER: 838 * Consider this function and ce_tasklet very tightly coupled pairs. 839 * Any changes to ce_tasklet or this function may likely need to be 840 * reflected in the counterpart. 841 * 842 * Returns: 843 * int: the amount of work done in this poll (<= budget) 844 */ 845 int hif_napi_poll(struct hif_opaque_softc *hif_ctx, 846 struct napi_struct *napi, 847 int budget) 848 { 849 int rc = 0; /* default: no work done, also takes care of error */ 850 int normalized = 0; 851 int bucket; 852 int cpu = smp_processor_id(); 853 bool poll_on_right_cpu; 854 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 855 struct qca_napi_info *napi_info; 856 struct CE_state *ce_state = NULL; 857 858 if (unlikely(NULL == hif)) { 859 HIF_ERROR("%s: hif context is NULL", __func__); 860 QDF_ASSERT(0); 861 goto out; 862 } 863 864 napi_info = (struct qca_napi_info *) 865 container_of(napi, struct qca_napi_info, napi); 866 867 NAPI_DEBUG("%s -->(napi(%d, irq=%d), budget=%d)", 868 __func__, napi_info->id, napi_info->irq, budget); 869 870 napi_info->stats[cpu].napi_polls++; 871 872 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 873 NAPI_POLL_ENTER, NULL, NULL, cpu, 0); 874 875 rc = ce_per_engine_service(hif, NAPI_ID2PIPE(napi_info->id)); 876 NAPI_DEBUG("%s: ce_per_engine_service processed %d msgs", 877 __func__, rc); 878 879 hif_napi_offld_flush_cb(napi_info); 880 881 /* do not return 0, if there was some work done, 882 * even if it is below the scale 883 */ 884 if (rc) { 885 napi_info->stats[cpu].napi_workdone += rc; 886 normalized = (rc / napi_info->scale); 887 if (normalized == 0) 888 normalized++; 889 bucket = normalized / (QCA_NAPI_BUDGET / QCA_NAPI_NUM_BUCKETS); 890 if (bucket >= QCA_NAPI_NUM_BUCKETS) { 891 bucket = QCA_NAPI_NUM_BUCKETS - 1; 892 HIF_ERROR("Bad bucket#(%d) > QCA_NAPI_NUM_BUCKETS(%d)", 893 bucket, QCA_NAPI_NUM_BUCKETS); 894 } 895 napi_info->stats[cpu].napi_budget_uses[bucket]++; 896 } else { 897 /* if ce_per engine reports 0, then poll should be terminated */ 898 NAPI_DEBUG("%s:%d: nothing processed by CE. Completing NAPI", 899 __func__, __LINE__); 900 } 901 902 ce_state = hif->ce_id_to_state[NAPI_ID2PIPE(napi_info->id)]; 903 904 /* 905 * Not using the API hif_napi_correct_cpu directly in the if statement 906 * below since the API may not get evaluated if put at the end if any 907 * prior condition would evaluate to be true. The CPU correction 908 * check should kick in every poll. 909 */ 910 #ifdef NAPI_YIELD_BUDGET_BASED 911 if (ce_state && (ce_state->force_break || 0 == rc)) { 912 #else 913 poll_on_right_cpu = hif_napi_correct_cpu(napi_info); 914 if ((ce_state) && 915 (!ce_check_rx_pending(ce_state) || (0 == rc) || 916 !poll_on_right_cpu)) { 917 #endif 918 napi_info->stats[cpu].napi_completes++; 919 #ifdef NAPI_YIELD_BUDGET_BASED 920 ce_state->force_break = 0; 921 #endif 922 923 hif_record_ce_desc_event(hif, ce_state->id, NAPI_COMPLETE, 924 NULL, NULL, 0, 0); 925 if (normalized >= budget) 926 normalized = budget - 1; 927 928 napi_complete(napi); 929 /* enable interrupts */ 930 hif_napi_enable_irq(hif_ctx, napi_info->id); 931 /* support suspend/resume */ 932 qdf_atomic_dec(&(hif->active_tasklet_cnt)); 933 934 NAPI_DEBUG("%s:%d: napi_complete + enabling the interrupts", 935 __func__, __LINE__); 936 } else { 937 /* 4.4 kernel NAPI implementation requires drivers to 938 * return full work when they ask to be re-scheduled, 939 * or napi_complete and re-start with a fresh interrupt 940 */ 941 normalized = budget; 942 } 943 944 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 945 NAPI_POLL_EXIT, NULL, NULL, normalized, 0); 946 947 NAPI_DEBUG("%s <--[normalized=%d]", __func__, normalized); 948 return normalized; 949 out: 950 return rc; 951 } 952 qdf_export_symbol(hif_napi_poll); 953 954 void hif_update_napi_max_poll_time(struct CE_state *ce_state, 955 struct qca_napi_info *napi_info, 956 int cpu_id) 957 { 958 unsigned long long napi_poll_time = sched_clock() - 959 ce_state->ce_service_start_time; 960 961 if (napi_poll_time > 962 napi_info->stats[cpu_id].napi_max_poll_time) 963 napi_info->stats[cpu_id].napi_max_poll_time = napi_poll_time; 964 } 965 966 #ifdef HIF_IRQ_AFFINITY 967 /** 968 * 969 * hif_napi_update_yield_stats() - update NAPI yield related stats 970 * @cpu_id: CPU ID for which stats needs to be updates 971 * @ce_id: Copy Engine ID for which yield stats needs to be updates 972 * @time_limit_reached: indicates whether the time limit was reached 973 * @rxpkt_thresh_reached: indicates whether rx packet threshold was reached 974 * 975 * Return: None 976 */ 977 void hif_napi_update_yield_stats(struct CE_state *ce_state, 978 bool time_limit_reached, 979 bool rxpkt_thresh_reached) 980 { 981 struct hif_softc *hif; 982 struct qca_napi_data *napi_data = NULL; 983 int ce_id = 0; 984 int cpu_id = 0; 985 986 if (unlikely(NULL == ce_state)) { 987 QDF_ASSERT(NULL != ce_state); 988 return; 989 } 990 991 hif = ce_state->scn; 992 993 if (unlikely(NULL == hif)) { 994 QDF_ASSERT(NULL != hif); 995 return; 996 } 997 napi_data = &(hif->napi_data); 998 if (unlikely(NULL == napi_data)) { 999 QDF_ASSERT(NULL != napi_data); 1000 return; 1001 } 1002 1003 ce_id = ce_state->id; 1004 cpu_id = qdf_get_cpu(); 1005 1006 if (unlikely(!napi_data->napis[ce_id])) { 1007 HIF_INFO("%s: NAPI info is NULL for ce id: %d", 1008 __func__, ce_id); 1009 return; 1010 } 1011 1012 if (time_limit_reached) 1013 napi_data->napis[ce_id]->stats[cpu_id].time_limit_reached++; 1014 else 1015 napi_data->napis[ce_id]->stats[cpu_id].rxpkt_thresh_reached++; 1016 1017 hif_update_napi_max_poll_time(ce_state, napi_data->napis[ce_id], 1018 cpu_id); 1019 } 1020 1021 /** 1022 * 1023 * hif_napi_stats() - display NAPI CPU statistics 1024 * @napid: pointer to qca_napi_data 1025 * 1026 * Description: 1027 * Prints the various CPU cores on which the NAPI instances /CEs interrupts 1028 * are being executed. Can be called from outside NAPI layer. 1029 * 1030 * Return: None 1031 */ 1032 void hif_napi_stats(struct qca_napi_data *napid) 1033 { 1034 int i; 1035 struct qca_napi_cpu *cpu; 1036 1037 if (napid == NULL) { 1038 qdf_debug("%s: napiid struct is null", __func__); 1039 return; 1040 } 1041 1042 cpu = napid->napi_cpu; 1043 qdf_debug("NAPI CPU TABLE"); 1044 qdf_debug("lilclhead=%d, bigclhead=%d", 1045 napid->lilcl_head, napid->bigcl_head); 1046 for (i = 0; i < NR_CPUS; i++) { 1047 qdf_debug("CPU[%02d]: state:%d crid=%02d clid=%02d crmk:0x%0lx thmk:0x%0lx frq:%d napi = 0x%08x lnk:%d", 1048 i, 1049 cpu[i].state, cpu[i].core_id, cpu[i].cluster_id, 1050 cpu[i].core_mask.bits[0], 1051 cpu[i].thread_mask.bits[0], 1052 cpu[i].max_freq, cpu[i].napis, 1053 cpu[i].cluster_nxt); 1054 } 1055 } 1056 1057 #ifdef FEATURE_NAPI_DEBUG 1058 /* 1059 * Local functions 1060 * - no argument checks, all internal/trusted callers 1061 */ 1062 static void hnc_dump_cpus(struct qca_napi_data *napid) 1063 { 1064 hif_napi_stats(napid); 1065 } 1066 #else 1067 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ }; 1068 #endif /* FEATURE_NAPI_DEBUG */ 1069 /** 1070 * hnc_link_clusters() - partitions to cpu table into clusters 1071 * @napid: pointer to NAPI data 1072 * 1073 * Takes in a CPU topology table and builds two linked lists 1074 * (big cluster cores, list-head at bigcl_head, and little cluster 1075 * cores, list-head at lilcl_head) out of it. 1076 * 1077 * If there are more than two clusters: 1078 * - bigcl_head and lilcl_head will be different, 1079 * - the cluster with highest cpufreq will be considered the "big" cluster. 1080 * If there are more than one with the highest frequency, the *last* of such 1081 * clusters will be designated as the "big cluster" 1082 * - the cluster with lowest cpufreq will be considered the "li'l" cluster. 1083 * If there are more than one clusters with the lowest cpu freq, the *first* 1084 * of such clusters will be designated as the "little cluster" 1085 * - We only support up to 32 clusters 1086 * Return: 0 : OK 1087 * !0: error (at least one of lil/big clusters could not be found) 1088 */ 1089 #define HNC_MIN_CLUSTER 0 1090 #define HNC_MAX_CLUSTER 31 1091 static int hnc_link_clusters(struct qca_napi_data *napid) 1092 { 1093 int rc = 0; 1094 1095 int i; 1096 int it = 0; 1097 uint32_t cl_done = 0x0; 1098 int cl, curcl, curclhead = 0; 1099 int more; 1100 unsigned int lilfrq = INT_MAX; 1101 unsigned int bigfrq = 0; 1102 unsigned int clfrq = 0; 1103 int prev = 0; 1104 struct qca_napi_cpu *cpus = napid->napi_cpu; 1105 1106 napid->lilcl_head = napid->bigcl_head = -1; 1107 1108 do { 1109 more = 0; 1110 it++; curcl = -1; 1111 for (i = 0; i < NR_CPUS; i++) { 1112 cl = cpus[i].cluster_id; 1113 NAPI_DEBUG("Processing cpu[%d], cluster=%d\n", 1114 i, cl); 1115 if ((cl < HNC_MIN_CLUSTER) || (cl > HNC_MAX_CLUSTER)) { 1116 NAPI_DEBUG("Bad cluster (%d). SKIPPED\n", cl); 1117 QDF_ASSERT(0); 1118 /* continue if ASSERTs are disabled */ 1119 continue; 1120 }; 1121 if (cpumask_weight(&(cpus[i].core_mask)) == 0) { 1122 NAPI_DEBUG("Core mask 0. SKIPPED\n"); 1123 continue; 1124 } 1125 if (cl_done & (0x01 << cl)) { 1126 NAPI_DEBUG("Cluster already processed. SKIPPED\n"); 1127 continue; 1128 } else { 1129 if (more == 0) { 1130 more = 1; 1131 curcl = cl; 1132 curclhead = i; /* row */ 1133 clfrq = cpus[i].max_freq; 1134 prev = -1; 1135 }; 1136 if ((curcl >= 0) && (curcl != cl)) { 1137 NAPI_DEBUG("Entry cl(%d) != curcl(%d). SKIPPED\n", 1138 cl, curcl); 1139 continue; 1140 } 1141 if (cpus[i].max_freq != clfrq) 1142 NAPI_DEBUG("WARN: frq(%d)!=clfrq(%d)\n", 1143 cpus[i].max_freq, clfrq); 1144 if (clfrq >= bigfrq) { 1145 bigfrq = clfrq; 1146 napid->bigcl_head = curclhead; 1147 NAPI_DEBUG("bigcl=%d\n", curclhead); 1148 } 1149 if (clfrq < lilfrq) { 1150 lilfrq = clfrq; 1151 napid->lilcl_head = curclhead; 1152 NAPI_DEBUG("lilcl=%d\n", curclhead); 1153 } 1154 if (prev != -1) 1155 cpus[prev].cluster_nxt = i; 1156 1157 prev = i; 1158 } 1159 } 1160 if (curcl >= 0) 1161 cl_done |= (0x01 << curcl); 1162 1163 } while (more); 1164 1165 if (qdf_unlikely((napid->lilcl_head < 0) && (napid->bigcl_head < 0))) 1166 rc = -EFAULT; 1167 1168 hnc_dump_cpus(napid); /* if NAPI_DEBUG */ 1169 return rc; 1170 } 1171 #undef HNC_MIN_CLUSTER 1172 #undef HNC_MAX_CLUSTER 1173 1174 /* 1175 * hotplug function group 1176 */ 1177 1178 /** 1179 * hnc_cpu_online_cb() - handles CPU hotplug "up" events 1180 * @context: the associated HIF context 1181 * @cpu: the CPU Id of the CPU the event happened on 1182 * 1183 * Return: None 1184 */ 1185 static void hnc_cpu_online_cb(void *context, uint32_t cpu) 1186 { 1187 struct hif_softc *hif = context; 1188 struct qca_napi_data *napid = &hif->napi_data; 1189 1190 if (cpu >= NR_CPUS) 1191 return; 1192 1193 NAPI_DEBUG("-->%s(act=online, cpu=%u)", __func__, cpu); 1194 1195 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_UP; 1196 NAPI_DEBUG("%s: CPU %u marked %d", 1197 __func__, cpu, napid->napi_cpu[cpu].state); 1198 1199 NAPI_DEBUG("<--%s", __func__); 1200 } 1201 1202 /** 1203 * hnc_cpu_before_offline_cb() - handles CPU hotplug "prepare down" events 1204 * @context: the associated HIF context 1205 * @cpu: the CPU Id of the CPU the event happened on 1206 * 1207 * On transtion to offline, we act on PREP events, because we may need to move 1208 * the irqs/NAPIs to another CPU before it is actually off-lined. 1209 * 1210 * Return: None 1211 */ 1212 static void hnc_cpu_before_offline_cb(void *context, uint32_t cpu) 1213 { 1214 struct hif_softc *hif = context; 1215 struct qca_napi_data *napid = &hif->napi_data; 1216 1217 if (cpu >= NR_CPUS) 1218 return; 1219 1220 NAPI_DEBUG("-->%s(act=before_offline, cpu=%u)", __func__, cpu); 1221 1222 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_DOWN; 1223 1224 NAPI_DEBUG("%s: CPU %u marked %d; updating affinity", 1225 __func__, cpu, napid->napi_cpu[cpu].state); 1226 1227 /** 1228 * we need to move any NAPIs on this CPU out. 1229 * if we are in LO throughput mode, then this is valid 1230 * if the CPU is the the low designated CPU. 1231 */ 1232 hif_napi_event(GET_HIF_OPAQUE_HDL(hif), 1233 NAPI_EVT_CPU_STATE, 1234 (void *) 1235 ((size_t)cpu << 16 | napid->napi_cpu[cpu].state)); 1236 1237 NAPI_DEBUG("<--%s", __func__); 1238 } 1239 1240 static int hnc_hotplug_register(struct hif_softc *hif_sc) 1241 { 1242 QDF_STATUS status; 1243 1244 NAPI_DEBUG("-->%s", __func__); 1245 1246 status = qdf_cpuhp_register(&hif_sc->napi_data.cpuhp_handler, 1247 hif_sc, 1248 hnc_cpu_online_cb, 1249 hnc_cpu_before_offline_cb); 1250 1251 NAPI_DEBUG("<--%s [%d]", __func__, status); 1252 1253 return qdf_status_to_os_return(status); 1254 } 1255 1256 static void hnc_hotplug_unregister(struct hif_softc *hif_sc) 1257 { 1258 NAPI_DEBUG("-->%s", __func__); 1259 1260 if (hif_sc->napi_data.cpuhp_handler) 1261 qdf_cpuhp_unregister(&hif_sc->napi_data.cpuhp_handler); 1262 1263 NAPI_DEBUG("<--%s", __func__); 1264 } 1265 1266 /** 1267 * hnc_install_tput() - installs a callback in the throughput detector 1268 * @register: !0 => register; =0: unregister 1269 * 1270 * installs a callback to be called when wifi driver throughput (tx+rx) 1271 * crosses a threshold. Currently, we are using the same criteria as 1272 * TCP ack suppression (500 packets/100ms by default). 1273 * 1274 * Return: 0 : success 1275 * <0: failure 1276 */ 1277 1278 static int hnc_tput_hook(int install) 1279 { 1280 int rc = 0; 1281 1282 /* 1283 * Nothing, until the bw_calculation accepts registration 1284 * it is now hardcoded in the wlan_hdd_main.c::hdd_bus_bw_compute_cbk 1285 * hdd_napi_throughput_policy(...) 1286 */ 1287 return rc; 1288 } 1289 1290 /* 1291 * Implementation of hif_napi_cpu API 1292 */ 1293 1294 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) 1295 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1296 { 1297 cpumask_copy(&(cpus[i].thread_mask), 1298 topology_sibling_cpumask(i)); 1299 } 1300 #else 1301 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1302 { 1303 } 1304 #endif 1305 1306 1307 /** 1308 * hif_napi_cpu_init() - initialization of irq affinity block 1309 * @ctx: pointer to qca_napi_data 1310 * 1311 * called by hif_napi_create, after the first instance is called 1312 * - builds napi_rss_cpus table from cpu topology 1313 * - links cores of the same clusters together 1314 * - installs hot-plug notifier 1315 * - installs throughput trigger notifier (when such mechanism exists) 1316 * 1317 * Return: 0: OK 1318 * <0: error code 1319 */ 1320 int hif_napi_cpu_init(struct hif_opaque_softc *hif) 1321 { 1322 int rc = 0; 1323 int i; 1324 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1325 struct qca_napi_cpu *cpus = napid->napi_cpu; 1326 1327 NAPI_DEBUG("--> "); 1328 1329 if (cpus[0].state != QCA_NAPI_CPU_UNINITIALIZED) { 1330 NAPI_DEBUG("NAPI RSS table already initialized.\n"); 1331 rc = -EALREADY; 1332 goto lab_rss_init; 1333 } 1334 1335 /* build CPU topology table */ 1336 for_each_possible_cpu(i) { 1337 cpus[i].state = ((cpumask_test_cpu(i, cpu_online_mask) 1338 ? QCA_NAPI_CPU_UP 1339 : QCA_NAPI_CPU_DOWN)); 1340 cpus[i].core_id = topology_core_id(i); 1341 cpus[i].cluster_id = topology_physical_package_id(i); 1342 cpumask_copy(&(cpus[i].core_mask), 1343 topology_core_cpumask(i)); 1344 record_sibling_cpumask(cpus, i); 1345 cpus[i].max_freq = cpufreq_quick_get_max(i); 1346 cpus[i].napis = 0x0; 1347 cpus[i].cluster_nxt = -1; /* invalid */ 1348 } 1349 1350 /* link clusters together */ 1351 rc = hnc_link_clusters(napid); 1352 if (0 != rc) 1353 goto lab_err_topology; 1354 1355 /* install hotplug notifier */ 1356 rc = hnc_hotplug_register(HIF_GET_SOFTC(hif)); 1357 if (0 != rc) 1358 goto lab_err_hotplug; 1359 1360 /* install throughput notifier */ 1361 rc = hnc_tput_hook(1); 1362 if (0 == rc) 1363 goto lab_rss_init; 1364 1365 lab_err_hotplug: 1366 hnc_tput_hook(0); 1367 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1368 lab_err_topology: 1369 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1370 lab_rss_init: 1371 NAPI_DEBUG("<-- [rc=%d]", rc); 1372 return rc; 1373 } 1374 1375 /** 1376 * hif_napi_cpu_deinit() - clean-up of irq affinity block 1377 * 1378 * called by hif_napi_destroy, when the last instance is removed 1379 * - uninstalls throughput and hotplug notifiers 1380 * - clears cpu topology table 1381 * Return: 0: OK 1382 */ 1383 int hif_napi_cpu_deinit(struct hif_opaque_softc *hif) 1384 { 1385 int rc = 0; 1386 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1387 1388 NAPI_DEBUG("-->%s(...)", __func__); 1389 1390 /* uninstall tput notifier */ 1391 rc = hnc_tput_hook(0); 1392 1393 /* uninstall hotplug notifier */ 1394 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1395 1396 /* clear the topology table */ 1397 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1398 1399 NAPI_DEBUG("<--%s[rc=%d]", __func__, rc); 1400 1401 return rc; 1402 } 1403 1404 /** 1405 * hncm_migrate_to() - migrates a NAPI to a CPU 1406 * @napid: pointer to NAPI block 1407 * @ce_id: CE_id of the NAPI instance 1408 * @didx : index in the CPU topology table for the CPU to migrate to 1409 * 1410 * Migrates NAPI (identified by the CE_id) to the destination core 1411 * Updates the napi_map of the destination entry 1412 * 1413 * Return: 1414 * =0 : success 1415 * <0 : error 1416 */ 1417 static int hncm_migrate_to(struct qca_napi_data *napid, 1418 int napi_ce, 1419 int didx) 1420 { 1421 int rc = 0; 1422 cpumask_t cpumask; 1423 1424 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, napi_ce, didx); 1425 1426 cpumask.bits[0] = (1 << didx); 1427 if (!napid->napis[napi_ce]) 1428 return -EINVAL; 1429 1430 irq_modify_status(napid->napis[napi_ce]->irq, IRQ_NO_BALANCING, 0); 1431 rc = irq_set_affinity_hint(napid->napis[napi_ce]->irq, &cpumask); 1432 1433 /* unmark the napis bitmap in the cpu table */ 1434 napid->napi_cpu[napid->napis[napi_ce]->cpu].napis &= ~(0x01 << napi_ce); 1435 /* mark the napis bitmap for the new designated cpu */ 1436 napid->napi_cpu[didx].napis |= (0x01 << napi_ce); 1437 napid->napis[napi_ce]->cpu = didx; 1438 1439 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1440 return rc; 1441 } 1442 /** 1443 * hncm_dest_cpu() - finds a destination CPU for NAPI 1444 * @napid: pointer to NAPI block 1445 * @act : RELOCATE | COLLAPSE | DISPERSE 1446 * 1447 * Finds the designated destionation for the next IRQ. 1448 * RELOCATE: translated to either COLLAPSE or DISPERSE based 1449 * on napid->napi_mode (throughput state) 1450 * COLLAPSE: All have the same destination: the first online CPU in lilcl 1451 * DISPERSE: One of the CPU in bigcl, which has the smallest number of 1452 * NAPIs on it 1453 * 1454 * Return: >=0 : index in the cpu topology table 1455 * : < 0 : error 1456 */ 1457 static int hncm_dest_cpu(struct qca_napi_data *napid, int act) 1458 { 1459 int destidx = -1; 1460 int head, i; 1461 1462 NAPI_DEBUG("-->%s(act=%d)", __func__, act); 1463 if (act == HNC_ACT_RELOCATE) { 1464 if (napid->napi_mode == QCA_NAPI_TPUT_LO) 1465 act = HNC_ACT_COLLAPSE; 1466 else 1467 act = HNC_ACT_DISPERSE; 1468 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d", 1469 __func__, act); 1470 } 1471 if (act == HNC_ACT_COLLAPSE) { 1472 head = i = napid->lilcl_head; 1473 retry_collapse: 1474 while (i >= 0) { 1475 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) { 1476 destidx = i; 1477 break; 1478 } 1479 i = napid->napi_cpu[i].cluster_nxt; 1480 } 1481 if ((destidx < 0) && (head == napid->lilcl_head)) { 1482 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl", 1483 __func__); 1484 head = i = napid->bigcl_head; 1485 goto retry_collapse; 1486 } 1487 } else { /* HNC_ACT_DISPERSE */ 1488 int smallest = 99; /* all 32 bits full */ 1489 int smallidx = -1; 1490 1491 head = i = napid->bigcl_head; 1492 retry_disperse: 1493 while (i >= 0) { 1494 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) && 1495 (hweight32(napid->napi_cpu[i].napis) <= smallest)) { 1496 smallest = napid->napi_cpu[i].napis; 1497 smallidx = i; 1498 } 1499 i = napid->napi_cpu[i].cluster_nxt; 1500 } 1501 destidx = smallidx; 1502 if ((destidx < 0) && (head == napid->bigcl_head)) { 1503 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl", 1504 __func__); 1505 head = i = napid->lilcl_head; 1506 goto retry_disperse; 1507 } 1508 } 1509 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx); 1510 return destidx; 1511 } 1512 /** 1513 * hif_napi_cpu_migrate() - migrate IRQs away 1514 * @cpu: -1: all CPUs <n> specific CPU 1515 * @act: COLLAPSE | DISPERSE 1516 * 1517 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible 1518 * cores. Eligible cores are: 1519 * act=COLLAPSE -> the first online core of the little cluster 1520 * act=DISPERSE -> separate cores of the big cluster, so that each core will 1521 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis) 1522 * 1523 * Note that this function is called with a spinlock acquired already. 1524 * 1525 * Return: =0: success 1526 * <0: error 1527 */ 1528 1529 int hif_napi_cpu_migrate(struct qca_napi_data *napid, int cpu, int action) 1530 { 1531 int rc = 0; 1532 struct qca_napi_cpu *cpup; 1533 int i, dind; 1534 uint32_t napis; 1535 1536 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)", 1537 __func__, cpu, action); 1538 /* the following is really: hif_napi_enabled() with less overhead */ 1539 if (napid->ce_map == 0) { 1540 NAPI_DEBUG("%s: NAPI disabled. Not migrating.", __func__); 1541 goto hncm_return; 1542 } 1543 1544 cpup = napid->napi_cpu; 1545 1546 switch (action) { 1547 case HNC_ACT_RELOCATE: 1548 case HNC_ACT_DISPERSE: 1549 case HNC_ACT_COLLAPSE: { 1550 /* first find the src napi set */ 1551 if (cpu == HNC_ANY_CPU) 1552 napis = napid->ce_map; 1553 else 1554 napis = cpup[cpu].napis; 1555 /* then clear the napi bitmap on each CPU */ 1556 for (i = 0; i < NR_CPUS; i++) 1557 cpup[i].napis = 0; 1558 /* then for each of the NAPIs to disperse: */ 1559 for (i = 0; i < CE_COUNT_MAX; i++) 1560 if (napis & (1 << i)) { 1561 /* find a destination CPU */ 1562 dind = hncm_dest_cpu(napid, action); 1563 if (dind >= 0) { 1564 NAPI_DEBUG("Migrating NAPI ce%d to %d", 1565 i, dind); 1566 rc = hncm_migrate_to(napid, i, dind); 1567 } else { 1568 NAPI_DEBUG("No dest for NAPI ce%d", i); 1569 hnc_dump_cpus(napid); 1570 rc = -1; 1571 } 1572 } 1573 break; 1574 } 1575 default: { 1576 NAPI_DEBUG("%s: bad action: %d\n", __func__, action); 1577 QDF_BUG(0); 1578 break; 1579 } 1580 } /* switch action */ 1581 1582 hncm_return: 1583 hnc_dump_cpus(napid); 1584 return rc; 1585 } 1586 1587 1588 /** 1589 * hif_napi_bl_irq() - calls irq_modify_status to enable/disable blacklisting 1590 * @napid: pointer to qca_napi_data structure 1591 * @bl_flag: blacklist flag to enable/disable blacklisting 1592 * 1593 * The function enables/disables blacklisting for all the copy engine 1594 * interrupts on which NAPI is enabled. 1595 * 1596 * Return: None 1597 */ 1598 static inline void hif_napi_bl_irq(struct qca_napi_data *napid, bool bl_flag) 1599 { 1600 int i; 1601 struct qca_napi_info *napii; 1602 1603 for (i = 0; i < CE_COUNT_MAX; i++) { 1604 /* check if NAPI is enabled on the CE */ 1605 if (!(napid->ce_map & (0x01 << i))) 1606 continue; 1607 1608 /*double check that NAPI is allocated for the CE */ 1609 napii = napid->napis[i]; 1610 if (!(napii)) 1611 continue; 1612 1613 if (bl_flag == true) 1614 irq_modify_status(napii->irq, 1615 0, IRQ_NO_BALANCING); 1616 else 1617 irq_modify_status(napii->irq, 1618 IRQ_NO_BALANCING, 0); 1619 HIF_DBG("%s: bl_flag %d CE %d", __func__, bl_flag, i); 1620 } 1621 } 1622 1623 #ifdef CONFIG_SCHED_CORE_CTL 1624 /* Enable this API only if kernel feature - CONFIG_SCHED_CORE_CTL is defined */ 1625 static inline int hif_napi_core_ctl_set_boost(bool boost) 1626 { 1627 return core_ctl_set_boost(boost); 1628 } 1629 #else 1630 static inline int hif_napi_core_ctl_set_boost(bool boost) 1631 { 1632 return 0; 1633 } 1634 #endif 1635 /** 1636 * hif_napi_cpu_blacklist() - en(dis)ables blacklisting for NAPI RX interrupts. 1637 * @napid: pointer to qca_napi_data structure 1638 * @op: blacklist operation to perform 1639 * 1640 * The function enables/disables/queries blacklisting for all CE RX 1641 * interrupts with NAPI enabled. Besides blacklisting, it also enables/disables 1642 * core_ctl_set_boost. 1643 * Once blacklisting is enabled, the interrupts will not be managed by the IRQ 1644 * balancer. 1645 * 1646 * Return: -EINVAL, in case IRQ_BLACKLISTING and CORE_CTL_BOOST is not enabled 1647 * for BLACKLIST_QUERY op - blacklist refcount 1648 * for BLACKLIST_ON op - return value from core_ctl_set_boost API 1649 * for BLACKLIST_OFF op - return value from core_ctl_set_boost API 1650 */ 1651 int hif_napi_cpu_blacklist(struct qca_napi_data *napid, 1652 enum qca_blacklist_op op) 1653 { 1654 int rc = 0; 1655 static int ref_count; /* = 0 by the compiler */ 1656 uint8_t flags = napid->flags; 1657 bool bl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING; 1658 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST; 1659 1660 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op); 1661 1662 if (!(bl_en && ccb_en)) { 1663 rc = -EINVAL; 1664 goto out; 1665 } 1666 1667 switch (op) { 1668 case BLACKLIST_QUERY: 1669 rc = ref_count; 1670 break; 1671 case BLACKLIST_ON: 1672 ref_count++; 1673 rc = 0; 1674 if (ref_count == 1) { 1675 rc = hif_napi_core_ctl_set_boost(true); 1676 NAPI_DEBUG("boost_on() returns %d - refcnt=%d", 1677 rc, ref_count); 1678 hif_napi_bl_irq(napid, true); 1679 } 1680 break; 1681 case BLACKLIST_OFF: 1682 if (ref_count) { 1683 ref_count--; 1684 rc = 0; 1685 if (ref_count == 0) { 1686 rc = hif_napi_core_ctl_set_boost(false); 1687 NAPI_DEBUG("boost_off() returns %d - refcnt=%d", 1688 rc, ref_count); 1689 hif_napi_bl_irq(napid, false); 1690 } 1691 } 1692 break; 1693 default: 1694 NAPI_DEBUG("Invalid blacklist op: %d", op); 1695 rc = -EINVAL; 1696 } /* switch */ 1697 out: 1698 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1699 return rc; 1700 } 1701 1702 /** 1703 * hif_napi_serialize() - [de-]serialize NAPI operations 1704 * @hif: context 1705 * @is_on: 1: serialize, 0: deserialize 1706 * 1707 * hif_napi_serialize(hif, 1) can be called multiple times. It will perform the 1708 * following steps (see hif_napi_event for code): 1709 * - put irqs of all NAPI instances on the same CPU 1710 * - only for the first serialize call: blacklist 1711 * 1712 * hif_napi_serialize(hif, 0): 1713 * - start a timer (multiple of BusBandwidthTimer -- default: 100 msec) 1714 * - at the end of the timer, check the current throughput state and 1715 * implement it. 1716 */ 1717 static unsigned long napi_serialize_reqs; 1718 int hif_napi_serialize(struct hif_opaque_softc *hif, int is_on) 1719 { 1720 int rc = -EINVAL; 1721 1722 if (hif != NULL) 1723 switch (is_on) { 1724 case 0: { /* de-serialize */ 1725 rc = hif_napi_event(hif, NAPI_EVT_USR_NORMAL, 1726 (void *) 0); 1727 napi_serialize_reqs = 0; 1728 break; 1729 } /* end de-serialize */ 1730 case 1: { /* serialize */ 1731 rc = hif_napi_event(hif, NAPI_EVT_USR_SERIAL, 1732 (void *)napi_serialize_reqs++); 1733 break; 1734 } /* end serialize */ 1735 default: 1736 break; /* no-op */ 1737 } /* switch */ 1738 return rc; 1739 } 1740 1741 #endif /* ifdef HIF_IRQ_AFFINITY */ 1742