1 /* 2 * Copyright (c) 2015-2021 The Linux Foundation. All rights reserved. 3 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for 6 * any purpose with or without fee is hereby granted, provided that the 7 * above copyright notice and this permission notice appear in all 8 * copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 17 * PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 /** 21 * DOC: hif_irq_affinity.c 22 * 23 * This irq affinity implementation is os dependent, so this can be treated as 24 * an abstraction layer... Should this be moved into a /linux folder? 25 */ 26 27 #include <linux/string.h> /* memset */ 28 29 /* Linux headers */ 30 #include <linux/cpumask.h> 31 #include <linux/cpufreq.h> 32 #include <linux/cpu.h> 33 #include <linux/topology.h> 34 #include <linux/interrupt.h> 35 #include <linux/pm.h> 36 #include <hif_napi.h> 37 #include <hif_irq_affinity.h> 38 #include <hif_exec.h> 39 #include <hif_main.h> 40 #include "qdf_irq.h" 41 42 #if defined(FEATURE_NAPI_DEBUG) && defined(HIF_IRQ_AFFINITY) 43 /* 44 * Local functions 45 * - no argument checks, all internal/trusted callers 46 */ 47 static void hnc_dump_cpus(struct qca_napi_data *napid) 48 { 49 hif_napi_stats(napid); 50 } 51 #else 52 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ }; 53 #endif /* FEATURE_NAPI_DEBUG */ 54 55 #ifdef HIF_IRQ_AFFINITY 56 /** 57 * 58 * hif_exec_event() - reacts to events that impact irq affinity 59 * @hif : pointer to hif context 60 * @evnt: event that has been detected 61 * @data: more data regarding the event 62 * 63 * Description: 64 * This function handles two types of events: 65 * 1- Events that change the state of NAPI (enabled/disabled): 66 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE} 67 * The state is retrievable by "hdd_napi_enabled(-1)" 68 * - NAPI will be on if either INI file is on and it has not been disabled 69 * by a subsequent vendor CMD, 70 * or it has been enabled by a vendor CMD. 71 * 2- Events that change the CPU affinity of a NAPI instance/IRQ: 72 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE} 73 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode 74 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy() 75 * - In LO tput mode, NAPI will yield control if its interrupts to the system 76 * management functions. However in HI throughput mode, NAPI will actively 77 * manage its interrupts/instances (by trying to disperse them out to 78 * separate performance cores). 79 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events. 80 * 81 * + In some cases (roaming peer management is the only case so far), a 82 * a client can trigger a "SERIALIZE" event. Basically, this means that the 83 * users is asking NAPI to go into a truly single execution context state. 84 * So, NAPI indicates to msm-irqbalancer that it wants to be denylisted, 85 * (if called for the first time) and then moves all IRQs (for NAPI 86 * instances) to be collapsed to a single core. If called multiple times, 87 * it will just re-collapse the CPUs. This is because denylist-on() API 88 * is reference-counted, and because the API has already been called. 89 * 90 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go 91 * to its "normal" operation. Optionally, they can give a timeout value (in 92 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this 93 * case, NAPI will just set the current throughput state to uninitialized 94 * and set the delay period. Once policy handler is called, it would skip 95 * applying the policy delay period times, and otherwise apply the policy. 96 * 97 * Return: 98 * < 0: some error 99 * = 0: event handled successfully 100 */ 101 int hif_exec_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event, 102 void *data) 103 { 104 int rc = 0; 105 uint32_t prev_state; 106 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 107 struct qca_napi_data *napid = &(hif->napi_data); 108 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED; 109 enum { 110 DENYLIST_NOT_PENDING, 111 DENYLIST_ON_PENDING, 112 DENYLIST_OFF_PENDING 113 } denylist_pending = DENYLIST_NOT_PENDING; 114 115 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data); 116 117 qdf_spin_lock_bh(&(napid->lock)); 118 prev_state = napid->state; 119 switch (event) { 120 case NAPI_EVT_INI_FILE: 121 case NAPI_EVT_CMD_STATE: 122 case NAPI_EVT_INT_STATE: 123 /* deprecated */ 124 break; 125 126 case NAPI_EVT_CPU_STATE: { 127 int cpu = ((unsigned long int)data >> 16); 128 int val = ((unsigned long int)data & 0x0ff); 129 130 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d", 131 __func__, cpu, val); 132 133 /* state has already been set by hnc_cpu_notify_cb */ 134 if ((val == QCA_NAPI_CPU_DOWN) && 135 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */ 136 (napid->napi_cpu[cpu].napis != 0)) { 137 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d", 138 __func__, cpu); 139 rc = hif_exec_cpu_migrate(napid, 140 cpu, 141 HNC_ACT_RELOCATE); 142 napid->napi_cpu[cpu].napis = 0; 143 } 144 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */ 145 break; 146 } 147 148 case NAPI_EVT_TPUT_STATE: { 149 tput_mode = (enum qca_napi_tput_state)data; 150 if (tput_mode == QCA_NAPI_TPUT_LO) { 151 /* from TPUT_HI -> TPUT_LO */ 152 NAPI_DEBUG("%s: Moving to napi_tput_LO state", 153 __func__); 154 denylist_pending = DENYLIST_OFF_PENDING; 155 /* 156 * Ideally we should "collapse" interrupts here, since 157 * we are "dispersing" interrupts in the "else" case. 158 * This allows the possibility that our interrupts may 159 * still be on the perf cluster the next time we enter 160 * high tput mode. However, the irq_balancer is free 161 * to move our interrupts to power cluster once 162 * denylisting has been turned off in the "else" case. 163 */ 164 } else { 165 /* from TPUT_LO -> TPUT->HI */ 166 NAPI_DEBUG("%s: Moving to napi_tput_HI state", 167 __func__); 168 rc = hif_exec_cpu_migrate(napid, 169 HNC_ANY_CPU, 170 HNC_ACT_DISPERSE); 171 172 denylist_pending = DENYLIST_ON_PENDING; 173 } 174 napid->napi_mode = tput_mode; 175 break; 176 } 177 178 case NAPI_EVT_USR_SERIAL: { 179 unsigned long users = (unsigned long)data; 180 181 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld", 182 __func__, users); 183 184 rc = hif_exec_cpu_migrate(napid, 185 HNC_ANY_CPU, 186 HNC_ACT_COLLAPSE); 187 if ((users == 0) && (rc == 0)) 188 denylist_pending = DENYLIST_ON_PENDING; 189 break; 190 } 191 case NAPI_EVT_USR_NORMAL: { 192 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__); 193 if (!napid->user_cpu_affin_mask) 194 denylist_pending = DENYLIST_OFF_PENDING; 195 /* 196 * Deserialization timeout is handled at hdd layer; 197 * just mark current mode to uninitialized to ensure 198 * it will be set when the delay is over 199 */ 200 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED; 201 break; 202 } 203 default: { 204 hif_err("Unknown event: %d (data=0x%0lx)", 205 event, (unsigned long) data); 206 break; 207 } /* default */ 208 }; /* switch */ 209 210 211 switch (denylist_pending) { 212 case DENYLIST_ON_PENDING: 213 /* assume the control of WLAN IRQs */ 214 hif_napi_cpu_denylist(napid, DENYLIST_ON); 215 break; 216 case DENYLIST_OFF_PENDING: 217 /* yield the control of WLAN IRQs */ 218 hif_napi_cpu_denylist(napid, DENYLIST_OFF); 219 break; 220 default: /* nothing to do */ 221 break; 222 } /* switch denylist_pending */ 223 224 qdf_spin_unlock_bh(&(napid->lock)); 225 226 NAPI_DEBUG("<--[rc=%d]", rc); 227 return rc; 228 } 229 230 #endif 231 232 /** 233 * hncm_migrate_to() - migrates a NAPI to a CPU 234 * @napid: pointer to NAPI block 235 * @ce_id: CE_id of the NAPI instance 236 * @didx : index in the CPU topology table for the CPU to migrate to 237 * 238 * Migrates NAPI (identified by the CE_id) to the destination core 239 * Updates the napi_map of the destination entry 240 * 241 * Return: 242 * =0 : success 243 * <0 : error 244 */ 245 static int hncm_exec_migrate_to(struct qca_napi_data *napid, uint8_t ctx_id, 246 int didx) 247 { 248 struct hif_exec_context *exec_ctx; 249 struct qdf_cpu_mask *cpumask; 250 int rc = 0; 251 int status = 0; 252 int ind; 253 254 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, ctx_id, didx); 255 256 exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, ctx_id); 257 if (!exec_ctx) 258 return -EINVAL; 259 260 exec_ctx->cpumask.bits[0] = (1 << didx); 261 262 for (ind = 0; ind < exec_ctx->numirq; ind++) { 263 if (exec_ctx->os_irq[ind]) { 264 qdf_dev_modify_irq_status(exec_ctx->os_irq[ind], 265 QDF_IRQ_NO_BALANCING, 0); 266 cpumask = (struct qdf_cpu_mask *)&exec_ctx->cpumask; 267 rc = qdf_dev_set_irq_affinity(exec_ctx->os_irq[ind], 268 cpumask); 269 if (rc) 270 status = rc; 271 } 272 } 273 274 /* unmark the napis bitmap in the cpu table */ 275 napid->napi_cpu[exec_ctx->cpu].napis &= ~(0x01 << ctx_id); 276 /* mark the napis bitmap for the new designated cpu */ 277 napid->napi_cpu[didx].napis |= (0x01 << ctx_id); 278 exec_ctx->cpu = didx; 279 280 NAPI_DEBUG("<--%s[%d]", __func__, rc); 281 return status; 282 } 283 284 /** 285 * hncm_dest_cpu() - finds a destination CPU for NAPI 286 * @napid: pointer to NAPI block 287 * @act : RELOCATE | COLLAPSE | DISPERSE 288 * 289 * Finds the designated destination for the next IRQ. 290 * RELOCATE: translated to either COLLAPSE or DISPERSE based 291 * on napid->napi_mode (throughput state) 292 * COLLAPSE: All have the same destination: the first online CPU in lilcl 293 * DISPERSE: One of the CPU in bigcl, which has the smallest number of 294 * NAPIs on it 295 * 296 * Return: >=0 : index in the cpu topology table 297 * : < 0 : error 298 */ 299 static int hncm_dest_cpu(struct qca_napi_data *napid, int act) 300 { 301 int destidx = -1; 302 int head, i; 303 304 NAPI_DEBUG("-->%s(act=%d)", __func__, act); 305 if (act == HNC_ACT_RELOCATE) { 306 if (napid->napi_mode == QCA_NAPI_TPUT_LO) 307 act = HNC_ACT_COLLAPSE; 308 else 309 act = HNC_ACT_DISPERSE; 310 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d", 311 __func__, act); 312 } 313 if (act == HNC_ACT_COLLAPSE) { 314 head = i = napid->lilcl_head; 315 retry_collapse: 316 while (i >= 0) { 317 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) { 318 destidx = i; 319 break; 320 } 321 i = napid->napi_cpu[i].cluster_nxt; 322 } 323 if ((destidx < 0) && (head == napid->lilcl_head)) { 324 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl", 325 __func__); 326 head = i = napid->bigcl_head; 327 goto retry_collapse; 328 } 329 } else { /* HNC_ACT_DISPERSE */ 330 int smallest = 99; /* all 32 bits full */ 331 int smallidx = -1; 332 333 head = i = napid->bigcl_head; 334 retry_disperse: 335 while (i >= 0) { 336 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) && 337 (hweight32(napid->napi_cpu[i].napis) <= smallest)) { 338 smallest = napid->napi_cpu[i].napis; 339 smallidx = i; 340 } 341 i = napid->napi_cpu[i].cluster_nxt; 342 } 343 destidx = smallidx; 344 if ((destidx < 0) && (head == napid->bigcl_head)) { 345 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl", 346 __func__); 347 head = i = napid->lilcl_head; 348 goto retry_disperse; 349 } 350 } 351 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx); 352 return destidx; 353 } 354 /** 355 * hif_napi_cpu_migrate() - migrate IRQs away 356 * @cpu: -1: all CPUs <n> specific CPU 357 * @act: COLLAPSE | DISPERSE 358 * 359 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible 360 * cores. Eligible cores are: 361 * act=COLLAPSE -> the first online core of the little cluster 362 * act=DISPERSE -> separate cores of the big cluster, so that each core will 363 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis) 364 * 365 * Note that this function is called with a spinlock acquired already. 366 * 367 * Return: =0: success 368 * <0: error 369 */ 370 int hif_exec_cpu_migrate(struct qca_napi_data *napid, int cpu, int action) 371 { 372 int rc = 0; 373 struct qca_napi_cpu *cpup; 374 int i, dind; 375 uint32_t napis; 376 377 378 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)", 379 __func__, cpu, action); 380 381 if (napid->exec_map == 0) { 382 NAPI_DEBUG("%s: datapath contexts to disperse", __func__); 383 goto hncm_return; 384 } 385 cpup = napid->napi_cpu; 386 387 switch (action) { 388 case HNC_ACT_RELOCATE: 389 case HNC_ACT_DISPERSE: 390 case HNC_ACT_COLLAPSE: { 391 /* first find the src napi set */ 392 if (cpu == HNC_ANY_CPU) 393 napis = napid->exec_map; 394 else 395 napis = cpup[cpu].napis; 396 /* then clear the napi bitmap on each CPU */ 397 for (i = 0; i < NR_CPUS; i++) 398 cpup[i].napis = 0; 399 /* then for each of the NAPIs to disperse: */ 400 for (i = 0; i < HIF_MAX_GROUP; i++) 401 if (napis & (1 << i)) { 402 /* find a destination CPU */ 403 dind = hncm_dest_cpu(napid, action); 404 if (dind >= 0) { 405 rc = hncm_exec_migrate_to(napid, i, 406 dind); 407 } else { 408 NAPI_DEBUG("No dest for NAPI ce%d", i); 409 hnc_dump_cpus(napid); 410 rc = -1; 411 } 412 } 413 break; 414 } 415 default: { 416 NAPI_DEBUG("%s: bad action: %d\n", __func__, action); 417 QDF_BUG(0); 418 break; 419 } 420 } /* switch action */ 421 422 hncm_return: 423 hnc_dump_cpus(napid); 424 return rc; 425 } 426 427 428 /** 429 * hif_exec_dl_irq() - calls irq_modify_status to enable/disable denylisting 430 * @napid: pointer to qca_napi_data structure 431 * @dl_flag: denylist flag to enable/disable denylisting 432 * 433 * The function enables/disables denylisting for all the copy engine 434 * interrupts on which NAPI is enabled. 435 * 436 * Return: None 437 */ 438 static inline void hif_exec_dl_irq(struct qca_napi_data *napid, bool dl_flag) 439 { 440 int i, j; 441 struct hif_exec_context *exec_ctx; 442 443 for (i = 0; i < HIF_MAX_GROUP; i++) { 444 /* check if NAPI is enabled on the CE */ 445 if (!(napid->exec_map & (0x01 << i))) 446 continue; 447 448 /*double check that NAPI is allocated for the CE */ 449 exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, i); 450 if (!(exec_ctx)) 451 continue; 452 453 if (dl_flag == true) 454 for (j = 0; j < exec_ctx->numirq; j++) 455 qdf_dev_modify_irq_status(exec_ctx->os_irq[j], 456 0, 457 QDF_IRQ_NO_BALANCING); 458 else 459 for (j = 0; j < exec_ctx->numirq; j++) 460 qdf_dev_modify_irq_status(exec_ctx->os_irq[j], 461 QDF_IRQ_NO_BALANCING, 462 0); 463 hif_debug("dl_flag %d CE %d", dl_flag, i); 464 } 465 } 466 467 /** 468 * hif_napi_cpu_denylist() - en(dis)ables denylisting for NAPI RX interrupts. 469 * @napid: pointer to qca_napi_data structure 470 * @op: denylist operation to perform 471 * 472 * The function enables/disables/queries denylisting for all CE RX 473 * interrupts with NAPI enabled. Besides denylisting, it also enables/disables 474 * core_ctl_set_boost. 475 * Once denylisting is enabled, the interrupts will not be managed by the IRQ 476 * balancer. 477 * 478 * Return: -EINVAL, in case IRQ_DENYLISTING and CORE_CTL_BOOST is not enabled 479 * for DENYLIST_QUERY op - denylist refcount 480 * for DENYLIST_ON op - return value from core_ctl_set_boost API 481 * for DENYLIST_OFF op - return value from core_ctl_set_boost API 482 */ 483 int hif_exec_cpu_denylist(struct qca_napi_data *napid, 484 enum qca_denylist_op op) 485 { 486 int rc = 0; 487 static int ref_count; /* = 0 by the compiler */ 488 uint8_t flags = napid->flags; 489 bool dl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING; 490 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST; 491 492 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op); 493 494 if (!(dl_en && ccb_en)) { 495 rc = -EINVAL; 496 goto out; 497 } 498 499 switch (op) { 500 case DENYLIST_QUERY: 501 rc = ref_count; 502 break; 503 case DENYLIST_ON: 504 ref_count++; 505 rc = 0; 506 if (ref_count == 1) { 507 rc = hif_napi_core_ctl_set_boost(true); 508 NAPI_DEBUG("boost_on() returns %d - refcnt=%d", 509 rc, ref_count); 510 hif_exec_dl_irq(napid, true); 511 } 512 break; 513 case DENYLIST_OFF: 514 if (ref_count) 515 ref_count--; 516 rc = 0; 517 if (ref_count == 0) { 518 rc = hif_napi_core_ctl_set_boost(false); 519 NAPI_DEBUG("boost_off() returns %d - refcnt=%d", 520 rc, ref_count); 521 hif_exec_dl_irq(napid, false); 522 } 523 break; 524 default: 525 NAPI_DEBUG("Invalid denylist op: %d", op); 526 rc = -EINVAL; 527 } /* switch */ 528 out: 529 NAPI_DEBUG("<--%s[%d]", __func__, rc); 530 return rc; 531 } 532 533