1 /* 2 * Copyright (c) 2014-2020 The Linux Foundation. All rights reserved. 3 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for 6 * any purpose with or without fee is hereby granted, provided that the 7 * above copyright notice and this permission notice appear in all 8 * copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 17 * PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <scheduler_core.h> 21 #include <qdf_atomic.h> 22 #include "qdf_flex_mem.h" 23 24 static struct scheduler_ctx g_sched_ctx; 25 static struct scheduler_ctx *gp_sched_ctx; 26 27 DEFINE_QDF_FLEX_MEM_POOL(sched_pool, sizeof(struct scheduler_msg), 28 WLAN_SCHED_REDUCTION_LIMIT); 29 30 #ifdef WLAN_SCHED_HISTORY_SIZE 31 32 #define SCHEDULER_HISTORY_HEADER "|Callback "\ 33 "|Message Type" \ 34 "|Queue Duration(us)|Queue Depth" \ 35 "|Run Duration(us)|" 36 37 #define SCHEDULER_HISTORY_LINE "--------------------------------------" \ 38 "--------------------------------------" \ 39 "--------------------------------------" 40 41 /** 42 * struct sched_history_item - metrics for a scheduler message 43 * @callback: the message's execution callback 44 * @type_id: the message's type_id 45 * @queue_id: Id of the queue the message was added to 46 * @queue_start_us: timestamp when the message was queued in microseconds 47 * @queue_duration_us: duration the message was queued in microseconds 48 * @queue_depth: depth of the queue when the message was queued 49 * @run_start_us: timesatmp when the message started execution in microseconds 50 * @run_duration_us: duration the message was executed in microseconds 51 */ 52 struct sched_history_item { 53 void *callback; 54 uint32_t type_id; 55 QDF_MODULE_ID queue_id; 56 uint64_t queue_start_us; 57 uint32_t queue_duration_us; 58 uint32_t queue_depth; 59 uint64_t run_start_us; 60 uint32_t run_duration_us; 61 }; 62 63 static struct sched_history_item sched_history[WLAN_SCHED_HISTORY_SIZE]; 64 static uint32_t sched_history_index; 65 66 static void sched_history_queue(struct scheduler_mq_type *queue, 67 struct scheduler_msg *msg) 68 { 69 msg->queue_id = queue->qid; 70 msg->queue_depth = qdf_list_size(&queue->mq_list); 71 msg->queued_at_us = qdf_get_log_timestamp_usecs(); 72 } 73 74 static void sched_history_start(struct scheduler_msg *msg) 75 { 76 uint64_t started_at_us = qdf_get_log_timestamp_usecs(); 77 struct sched_history_item hist = { 78 .callback = msg->callback, 79 .type_id = msg->type, 80 .queue_start_us = msg->queued_at_us, 81 .queue_duration_us = started_at_us - msg->queued_at_us, 82 .queue_depth = msg->queue_depth, 83 .run_start_us = started_at_us, 84 }; 85 86 sched_history[sched_history_index] = hist; 87 } 88 89 static void sched_history_stop(void) 90 { 91 struct sched_history_item *hist = &sched_history[sched_history_index]; 92 uint64_t stopped_at_us = qdf_get_log_timestamp_usecs(); 93 94 hist->run_duration_us = stopped_at_us - hist->run_start_us; 95 96 sched_history_index++; 97 sched_history_index %= WLAN_SCHED_HISTORY_SIZE; 98 } 99 100 void sched_history_print(void) 101 { 102 struct sched_history_item *history, *item; 103 uint32_t history_idx; 104 uint32_t idx, index; 105 106 history = qdf_mem_malloc(sizeof(*history) * WLAN_SCHED_HISTORY_SIZE); 107 108 if (!history) { 109 sched_err("Mem alloc failed"); 110 return; 111 } 112 113 qdf_mem_copy(history, &sched_history, 114 (sizeof(*history) * WLAN_SCHED_HISTORY_SIZE)); 115 history_idx = sched_history_index; 116 117 sched_nofl_fatal(SCHEDULER_HISTORY_LINE); 118 sched_nofl_fatal(SCHEDULER_HISTORY_HEADER); 119 sched_nofl_fatal(SCHEDULER_HISTORY_LINE); 120 121 for (idx = 0; idx < WLAN_SCHED_HISTORY_SIZE; idx++) { 122 index = (history_idx + idx) % WLAN_SCHED_HISTORY_SIZE; 123 item = history + index; 124 125 if (!item->callback) 126 continue; 127 128 sched_nofl_fatal("%40pF|%12d|%18d|%11d|%16d|", 129 item->callback, item->type_id, 130 item->queue_duration_us, 131 item->queue_depth, 132 item->run_duration_us); 133 } 134 135 sched_nofl_fatal(SCHEDULER_HISTORY_LINE); 136 137 qdf_mem_free(history); 138 } 139 #else /* WLAN_SCHED_HISTORY_SIZE */ 140 141 static inline void sched_history_queue(struct scheduler_mq_type *queue, 142 struct scheduler_msg *msg) { } 143 static inline void sched_history_start(struct scheduler_msg *msg) { } 144 static inline void sched_history_stop(void) { } 145 void sched_history_print(void) { } 146 147 #endif /* WLAN_SCHED_HISTORY_SIZE */ 148 149 QDF_STATUS scheduler_create_ctx(void) 150 { 151 qdf_flex_mem_init(&sched_pool); 152 gp_sched_ctx = &g_sched_ctx; 153 154 return QDF_STATUS_SUCCESS; 155 } 156 157 QDF_STATUS scheduler_destroy_ctx(void) 158 { 159 gp_sched_ctx = NULL; 160 qdf_flex_mem_deinit(&sched_pool); 161 162 return QDF_STATUS_SUCCESS; 163 } 164 165 struct scheduler_ctx *scheduler_get_context(void) 166 { 167 QDF_BUG(gp_sched_ctx); 168 169 return gp_sched_ctx; 170 } 171 172 static QDF_STATUS scheduler_mq_init(struct scheduler_mq_type *msg_q) 173 { 174 sched_enter(); 175 176 qdf_spinlock_create(&msg_q->mq_lock); 177 qdf_list_create(&msg_q->mq_list, SCHEDULER_CORE_MAX_MESSAGES); 178 179 sched_exit(); 180 181 return QDF_STATUS_SUCCESS; 182 } 183 184 static void scheduler_mq_deinit(struct scheduler_mq_type *msg_q) 185 { 186 sched_enter(); 187 188 qdf_list_destroy(&msg_q->mq_list); 189 qdf_spinlock_destroy(&msg_q->mq_lock); 190 191 sched_exit(); 192 } 193 194 static qdf_atomic_t __sched_queue_depth; 195 static qdf_atomic_t __sched_dup_fail_count; 196 197 static QDF_STATUS scheduler_all_queues_init(struct scheduler_ctx *sched_ctx) 198 { 199 QDF_STATUS status; 200 int i; 201 202 sched_enter(); 203 204 QDF_BUG(sched_ctx); 205 if (!sched_ctx) 206 return QDF_STATUS_E_FAILURE; 207 208 qdf_atomic_set(&__sched_queue_depth, 0); 209 210 /* Initialize all message queues */ 211 for (i = 0; i < SCHEDULER_NUMBER_OF_MSG_QUEUE; i++) { 212 status = scheduler_mq_init(&sched_ctx->queue_ctx.sch_msg_q[i]); 213 if (QDF_STATUS_SUCCESS != status) 214 return status; 215 } 216 217 /* Initialize all qid to qidx mapping to invalid values */ 218 for (i = 0; i < QDF_MODULE_ID_MAX; i++) 219 sched_ctx->queue_ctx.scheduler_msg_qid_to_qidx[i] = 220 SCHEDULER_NUMBER_OF_MSG_QUEUE; 221 222 sched_exit(); 223 224 return status; 225 } 226 227 static QDF_STATUS scheduler_all_queues_deinit(struct scheduler_ctx *sched_ctx) 228 { 229 int i; 230 231 sched_enter(); 232 233 QDF_BUG(sched_ctx); 234 if (!sched_ctx) 235 return QDF_STATUS_E_FAILURE; 236 237 /* De-Initialize all message queues */ 238 for (i = 0; i < SCHEDULER_NUMBER_OF_MSG_QUEUE; i++) 239 scheduler_mq_deinit(&sched_ctx->queue_ctx.sch_msg_q[i]); 240 241 /* Initialize all qid to qidx mapping to invalid values */ 242 for (i = 0; i < QDF_MODULE_ID_MAX; i++) 243 sched_ctx->queue_ctx.scheduler_msg_qid_to_qidx[i] = 244 SCHEDULER_NUMBER_OF_MSG_QUEUE; 245 246 sched_exit(); 247 248 return QDF_STATUS_SUCCESS; 249 } 250 251 void scheduler_mq_put(struct scheduler_mq_type *msg_q, 252 struct scheduler_msg *msg) 253 { 254 qdf_spin_lock_irqsave(&msg_q->mq_lock); 255 sched_history_queue(msg_q, msg); 256 qdf_list_insert_back(&msg_q->mq_list, &msg->node); 257 qdf_spin_unlock_irqrestore(&msg_q->mq_lock); 258 } 259 260 void scheduler_mq_put_front(struct scheduler_mq_type *msg_q, 261 struct scheduler_msg *msg) 262 { 263 qdf_spin_lock_irqsave(&msg_q->mq_lock); 264 sched_history_queue(msg_q, msg); 265 qdf_list_insert_front(&msg_q->mq_list, &msg->node); 266 qdf_spin_unlock_irqrestore(&msg_q->mq_lock); 267 } 268 269 struct scheduler_msg *scheduler_mq_get(struct scheduler_mq_type *msg_q) 270 { 271 QDF_STATUS status; 272 qdf_list_node_t *node; 273 274 qdf_spin_lock_irqsave(&msg_q->mq_lock); 275 status = qdf_list_remove_front(&msg_q->mq_list, &node); 276 qdf_spin_unlock_irqrestore(&msg_q->mq_lock); 277 278 if (QDF_IS_STATUS_ERROR(status)) 279 return NULL; 280 281 return qdf_container_of(node, struct scheduler_msg, node); 282 } 283 284 QDF_STATUS scheduler_queues_deinit(struct scheduler_ctx *sched_ctx) 285 { 286 return scheduler_all_queues_deinit(sched_ctx); 287 } 288 289 QDF_STATUS scheduler_queues_init(struct scheduler_ctx *sched_ctx) 290 { 291 QDF_STATUS status; 292 293 sched_enter(); 294 295 QDF_BUG(sched_ctx); 296 if (!sched_ctx) 297 return QDF_STATUS_E_FAILURE; 298 299 status = scheduler_all_queues_init(sched_ctx); 300 if (QDF_IS_STATUS_ERROR(status)) { 301 scheduler_all_queues_deinit(sched_ctx); 302 sched_err("Failed to initialize the msg queues"); 303 return status; 304 } 305 306 sched_debug("Queue init passed"); 307 308 sched_exit(); 309 310 return QDF_STATUS_SUCCESS; 311 } 312 313 struct scheduler_msg *scheduler_core_msg_dup(struct scheduler_msg *msg) 314 { 315 struct scheduler_msg *dup; 316 317 if (qdf_atomic_inc_return(&__sched_queue_depth) > 318 SCHEDULER_CORE_MAX_MESSAGES) 319 goto buffer_full; 320 321 dup = qdf_flex_mem_alloc(&sched_pool); 322 if (!dup) { 323 sched_err("out of memory"); 324 goto dec_queue_count; 325 } 326 327 qdf_mem_copy(dup, msg, sizeof(*dup)); 328 329 qdf_atomic_set(&__sched_dup_fail_count, 0); 330 331 return dup; 332 333 buffer_full: 334 if (qdf_atomic_inc_return(&__sched_dup_fail_count) > 335 SCHEDULER_WRAPPER_MAX_FAIL_COUNT) 336 QDF_DEBUG_PANIC("Scheduler buffer is full"); 337 338 339 dec_queue_count: 340 qdf_atomic_dec(&__sched_queue_depth); 341 342 return NULL; 343 } 344 345 void scheduler_core_msg_free(struct scheduler_msg *msg) 346 { 347 qdf_flex_mem_free(&sched_pool, msg); 348 qdf_atomic_dec(&__sched_queue_depth); 349 } 350 351 static void scheduler_thread_process_queues(struct scheduler_ctx *sch_ctx, 352 bool *shutdown) 353 { 354 int i; 355 QDF_STATUS status; 356 struct scheduler_msg *msg; 357 358 if (!sch_ctx) { 359 QDF_DEBUG_PANIC("sch_ctx is null"); 360 return; 361 } 362 363 /* start with highest priority queue : timer queue at index 0 */ 364 i = 0; 365 while (i < SCHEDULER_NUMBER_OF_MSG_QUEUE) { 366 /* Check if MC needs to shutdown */ 367 if (qdf_atomic_test_bit(MC_SHUTDOWN_EVENT_MASK, 368 &sch_ctx->sch_event_flag)) { 369 sched_debug("scheduler thread signaled to shutdown"); 370 *shutdown = true; 371 372 /* Check for any Suspend Indication */ 373 if (qdf_atomic_test_and_clear_bit(MC_SUSPEND_EVENT_MASK, 374 &sch_ctx->sch_event_flag)) { 375 /* Unblock anyone waiting on suspend */ 376 if (gp_sched_ctx->hdd_callback) 377 gp_sched_ctx->hdd_callback(); 378 } 379 380 break; 381 } 382 383 msg = scheduler_mq_get(&sch_ctx->queue_ctx.sch_msg_q[i]); 384 if (!msg) { 385 /* check next queue */ 386 i++; 387 continue; 388 } 389 390 if (sch_ctx->queue_ctx.scheduler_msg_process_fn[i]) { 391 sch_ctx->watchdog_msg_type = msg->type; 392 sch_ctx->watchdog_callback = msg->callback; 393 394 sched_history_start(msg); 395 qdf_timer_start(&sch_ctx->watchdog_timer, 396 sch_ctx->timeout); 397 status = sch_ctx->queue_ctx. 398 scheduler_msg_process_fn[i](msg); 399 qdf_timer_stop(&sch_ctx->watchdog_timer); 400 sched_history_stop(); 401 402 if (QDF_IS_STATUS_ERROR(status)) 403 sched_err("Failed processing Qid[%d] message", 404 sch_ctx->queue_ctx.sch_msg_q[i].qid); 405 406 scheduler_core_msg_free(msg); 407 } 408 409 /* start again with highest priority queue at index 0 */ 410 i = 0; 411 } 412 413 /* Check for any Suspend Indication */ 414 if (qdf_atomic_test_and_clear_bit(MC_SUSPEND_EVENT_MASK, 415 &sch_ctx->sch_event_flag)) { 416 qdf_spin_lock(&sch_ctx->sch_thread_lock); 417 qdf_event_reset(&sch_ctx->resume_sch_event); 418 /* controller thread suspend completion callback */ 419 if (gp_sched_ctx->hdd_callback) 420 gp_sched_ctx->hdd_callback(); 421 qdf_spin_unlock(&sch_ctx->sch_thread_lock); 422 /* Wait for resume indication */ 423 qdf_wait_single_event(&sch_ctx->resume_sch_event, 0); 424 } 425 426 return; /* Nothing to process wait on wait queue */ 427 } 428 429 int scheduler_thread(void *arg) 430 { 431 struct scheduler_ctx *sch_ctx = (struct scheduler_ctx *)arg; 432 int retWaitStatus = 0; 433 bool shutdown = false; 434 435 if (!arg) { 436 QDF_DEBUG_PANIC("arg is null"); 437 return 0; 438 } 439 qdf_set_user_nice(current, -2); 440 441 /* Ack back to the context from which the main controller thread 442 * has been created 443 */ 444 qdf_event_set(&sch_ctx->sch_start_event); 445 sched_debug("scheduler thread %d (%s) starting up", 446 current->pid, current->comm); 447 448 while (!shutdown) { 449 /* This implements the execution model algorithm */ 450 retWaitStatus = qdf_wait_queue_interruptible( 451 sch_ctx->sch_wait_queue, 452 qdf_atomic_test_bit(MC_POST_EVENT_MASK, 453 &sch_ctx->sch_event_flag) || 454 qdf_atomic_test_bit(MC_SUSPEND_EVENT_MASK, 455 &sch_ctx->sch_event_flag)); 456 457 if (retWaitStatus == -ERESTARTSYS) 458 QDF_DEBUG_PANIC("Scheduler received -ERESTARTSYS"); 459 460 qdf_atomic_clear_bit(MC_POST_EVENT_MASK, &sch_ctx->sch_event_flag); 461 scheduler_thread_process_queues(sch_ctx, &shutdown); 462 } 463 464 /* If we get here the scheduler thread must exit */ 465 sched_debug("Scheduler thread exiting"); 466 qdf_event_set(&sch_ctx->sch_shutdown); 467 468 return 0; 469 } 470 471 static void scheduler_flush_single_queue(struct scheduler_mq_type *mq) 472 { 473 struct scheduler_msg *msg; 474 QDF_STATUS (*flush_cb)(struct scheduler_msg *); 475 476 while ((msg = scheduler_mq_get(mq))) { 477 if (msg->flush_callback) { 478 sched_debug("Calling flush callback; type: %x", 479 msg->type); 480 flush_cb = msg->flush_callback; 481 flush_cb(msg); 482 } else if (msg->bodyptr) { 483 sched_debug("Freeing scheduler msg bodyptr; type: %x", 484 msg->type); 485 qdf_mem_free(msg->bodyptr); 486 } 487 488 scheduler_core_msg_free(msg); 489 } 490 } 491 492 void scheduler_queues_flush(struct scheduler_ctx *sched_ctx) 493 { 494 struct scheduler_mq_type *mq; 495 int i; 496 497 sched_debug("Flushing scheduler message queues"); 498 499 for (i = 0; i < SCHEDULER_NUMBER_OF_MSG_QUEUE; i++) { 500 mq = &sched_ctx->queue_ctx.sch_msg_q[i]; 501 scheduler_flush_single_queue(mq); 502 } 503 } 504 505