241 	void (*enqueue)(struct task_struct *p, u64 enq_flags);
317 	void (*runnable)(struct task_struct *p, u64 enq_flags);
963 	u64			enq_flags;  member
1672 			     u64 enq_flags)  in dispatch_enqueue()  argument
1692 		     (enq_flags & SCX_ENQ_DSQ_PRIQ))) {  in dispatch_enqueue()
1701 		enq_flags &= ~SCX_ENQ_DSQ_PRIQ;  in dispatch_enqueue()
1704 	if (enq_flags & SCX_ENQ_DSQ_PRIQ) {  in dispatch_enqueue()
1739 		if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))  in dispatch_enqueue()
1765 	if (enq_flags & SCX_ENQ_CLEAR_OPSS)  in dispatch_enqueue()
1772 		if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&  in dispatch_enqueue()
1885 				 u64 enq_flags)  in mark_direct_dispatch()  argument
1910 	p->scx.ddsp_enq_flags = enq_flags;  in mark_direct_dispatch()
1913 static void direct_dispatch(struct task_struct *p, u64 enq_flags)  in direct_dispatch()  argument
1921 	p->scx.ddsp_enq_flags |= enq_flags;  in direct_dispatch()
1973 static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,  in do_enqueue_task()  argument
2017 	SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags);  in do_enqueue_task()
2031 	direct_dispatch(p, enq_flags);  in do_enqueue_task()
2043 	dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags);  in do_enqueue_task()
2049 	dispatch_enqueue(find_global_dsq(p), p, enq_flags);  in do_enqueue_task()
2080 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags)  in enqueue_task_scx()  argument
2084 	if (enq_flags & ENQUEUE_WAKEUP)  in enqueue_task_scx()
2087 	enq_flags |= rq->scx.extra_enq_flags;  in enqueue_task_scx()
2098 	if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p))  in enqueue_task_scx()
2112 		SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags);  in enqueue_task_scx()
2114 	if (enq_flags & SCX_ENQ_WAKEUP)  in enqueue_task_scx()
2117 	do_enqueue_task(rq, p, enq_flags, sticky_cpu);  in enqueue_task_scx()
2231 static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,  in move_local_task_to_local_dsq()  argument
2243 	if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))  in move_local_task_to_local_dsq()
2262 static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,  in move_remote_task_to_local_dsq()  argument
2282 	dst_rq->scx.extra_enq_flags = enq_flags;  in move_remote_task_to_local_dsq()
2395 static inline void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *s…  in move_remote_task_to_local_dsq()  argument
2457 				  struct task_struct *p, u64 enq_flags)  in dispatch_to_local_dsq()  argument
2469 		dispatch_enqueue(dst_dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);  in dispatch_to_local_dsq()
2476 				 enq_flags | SCX_ENQ_CLEAR_OPSS);  in dispatch_to_local_dsq()
2512 			dispatch_enqueue(&dst_rq->scx.local_dsq, p, enq_flags);  in dispatch_to_local_dsq()
2514 			move_remote_task_to_local_dsq(p, enq_flags,  in dispatch_to_local_dsq()
2554 			    u64 dsq_id, u64 enq_flags)  in finish_dispatch()  argument
2608 		dispatch_to_local_dsq(rq, dsq, p, enq_flags);  in finish_dispatch()
2610 		dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);  in finish_dispatch()
2622 				ent->enq_flags);  in flush_dispatch_buf()
5545 static void enqueue_stub(struct task_struct *p, u64 enq_flags) {}  in enqueue_stub()  argument
5546 static void dequeue_stub(struct task_struct *p, u64 enq_flags) {}  in dequeue_stub()  argument
5549 static void runnable_stub(struct task_struct *p, u64 enq_flags) {}  in runnable_stub()  argument
5947 static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags)  in scx_dispatch_preamble()  argument
5959 	if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) {  in scx_dispatch_preamble()
5960 		scx_ops_error("invalid enq_flags 0x%llx", enq_flags);  in scx_dispatch_preamble()
5967 static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags)  in scx_dispatch_commit()  argument
5974 		mark_direct_dispatch(ddsp_task, p, dsq_id, enq_flags);  in scx_dispatch_commit()
5987 		.enq_flags = enq_flags,  in scx_dispatch_commit()
6030 				  u64 enq_flags)  in scx_bpf_dispatch()  argument
6032 	if (!scx_dispatch_preamble(p, enq_flags))  in scx_bpf_dispatch()
6040 	scx_dispatch_commit(p, dsq_id, enq_flags);  in scx_bpf_dispatch()
6061 					u64 slice, u64 vtime, u64 enq_flags)  in scx_bpf_dispatch_vtime()  argument
6063 	if (!scx_dispatch_preamble(p, enq_flags))  in scx_bpf_dispatch_vtime()
6073 	scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);  in scx_bpf_dispatch_vtime()
6090 				  u64 enq_flags)  in scx_dispatch_from_dsq()  argument
6158 			move_local_task_to_local_dsq(p, enq_flags,  in scx_dispatch_from_dsq()
6163 			move_remote_task_to_local_dsq(p, enq_flags,  in scx_dispatch_from_dsq()
6178 		dispatch_enqueue(dst_dsq, p, enq_flags);  in scx_dispatch_from_dsq()
6343 					   u64 enq_flags)  in scx_bpf_dispatch_from_dsq()  argument
6346 				     p, dsq_id, enq_flags);  in scx_bpf_dispatch_from_dsq()
6369 						 u64 enq_flags)  in scx_bpf_dispatch_vtime_from_dsq()  argument
6372 				     p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);  in scx_bpf_dispatch_vtime_from_dsq()