1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef KFD_MQD_MANAGER_H_ 26 #define KFD_MQD_MANAGER_H_ 27 28 #include "kfd_priv.h" 29 30 #define KFD_MAX_NUM_SE 8 31 #define KFD_MAX_NUM_SH_PER_SE 2 32 33 /** 34 * struct mqd_manager 35 * 36 * @init_mqd: Allocates the mqd buffer on local gpu memory and initialize it. 37 * 38 * @load_mqd: Loads the mqd to a concrete hqd slot. Used only for no cp 39 * scheduling mode. 40 * 41 * @update_mqd: Handles a update call for the MQD 42 * 43 * @destroy_mqd: Destroys the HQD slot and by that preempt the relevant queue. 44 * Used only for no cp scheduling. 45 * 46 * @free_mqd: Releases the mqd buffer from local gpu memory. 47 * 48 * @is_occupied: Checks if the relevant HQD slot is occupied. 49 * 50 * @get_wave_state: Retrieves context save state and optionally copies the 51 * control stack, if kept in the MQD, to the given userspace address. 52 * 53 * @mqd_mutex: Mqd manager mutex. 54 * 55 * @dev: The kfd device structure coupled with this module. 56 * 57 * MQD stands for Memory Queue Descriptor which represents the current queue 58 * state in the memory and initiate the HQD (Hardware Queue Descriptor) state. 59 * This structure is actually a base class for the different types of MQDs 60 * structures for the variant ASICs that should be supported in the future. 61 * This base class is also contains all the MQD specific operations. 62 * Another important thing to mention is that each queue has a MQD that keeps 63 * his state (or context) after each preemption or reassignment. 64 * Basically there are a instances of the mqd manager class per MQD type per 65 * ASIC. Currently the kfd driver supports only Kaveri so there are instances 66 * per KFD_MQD_TYPE for each device. 67 * 68 */ 69 extern int pipe_priority_map[]; 70 struct mqd_manager { 71 struct kfd_mem_obj* (*allocate_mqd)(struct kfd_node *kfd, 72 struct queue_properties *q); 73 74 void (*init_mqd)(struct mqd_manager *mm, void **mqd, 75 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 76 struct queue_properties *q); 77 78 int (*load_mqd)(struct mqd_manager *mm, void *mqd, 79 uint32_t pipe_id, uint32_t queue_id, 80 struct queue_properties *p, 81 struct mm_struct *mms); 82 83 void (*update_mqd)(struct mqd_manager *mm, void *mqd, 84 struct queue_properties *q, 85 struct mqd_update_info *minfo); 86 87 int (*destroy_mqd)(struct mqd_manager *mm, void *mqd, 88 enum kfd_preempt_type type, 89 unsigned int timeout, uint32_t pipe_id, 90 uint32_t queue_id); 91 92 void (*free_mqd)(struct mqd_manager *mm, void *mqd, 93 struct kfd_mem_obj *mqd_mem_obj); 94 95 bool (*is_occupied)(struct mqd_manager *mm, void *mqd, 96 uint64_t queue_address, uint32_t pipe_id, 97 uint32_t queue_id); 98 99 int (*get_wave_state)(struct mqd_manager *mm, void *mqd, 100 struct queue_properties *q, 101 void __user *ctl_stack, 102 u32 *ctl_stack_used_size, 103 u32 *save_area_used_size); 104 105 void (*get_checkpoint_info)(struct mqd_manager *mm, void *mqd, uint32_t *ctl_stack_size); 106 107 void (*checkpoint_mqd)(struct mqd_manager *mm, 108 void *mqd, 109 void *mqd_dst, 110 void *ctl_stack_dst); 111 112 void (*restore_mqd)(struct mqd_manager *mm, void **mqd, 113 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 114 struct queue_properties *p, 115 const void *mqd_src, 116 const void *ctl_stack_src, 117 const u32 ctl_stack_size); 118 119 #if defined(CONFIG_DEBUG_FS) 120 int (*debugfs_show_mqd)(struct seq_file *m, void *data); 121 #endif 122 bool (*check_preemption_failed)(struct mqd_manager *mm, void *mqd); 123 uint64_t (*mqd_stride)(struct mqd_manager *mm, 124 struct queue_properties *p); 125 126 struct mutex mqd_mutex; 127 struct kfd_node *dev; 128 uint32_t mqd_size; 129 }; 130 131 struct mqd_user_context_save_area_header { 132 /* Byte offset from start of user context 133 * save area to the last saved top (lowest 134 * address) of control stack data. Must be 135 * 4 byte aligned. 136 */ 137 uint32_t control_stack_offset; 138 139 /* Byte size of the last saved control stack 140 * data. Must be 4 byte aligned. 141 */ 142 uint32_t control_stack_size; 143 144 /* Byte offset from start of user context save 145 * area to the last saved base (lowest address) 146 * of wave state data. Must be 4 byte aligned. 147 */ 148 uint32_t wave_state_offset; 149 150 /* Byte size of the last saved wave state data. 151 * Must be 4 byte aligned. 152 */ 153 uint32_t wave_state_size; 154 }; 155 156 struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_node *dev, 157 struct queue_properties *q); 158 159 struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_node *dev, 160 struct queue_properties *q); 161 void free_mqd_hiq_sdma(struct mqd_manager *mm, void *mqd, 162 struct kfd_mem_obj *mqd_mem_obj); 163 164 void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm, 165 const uint32_t *cu_mask, uint32_t cu_mask_count, 166 uint32_t *se_mask, uint32_t inst); 167 168 int kfd_hiq_load_mqd_kiq(struct mqd_manager *mm, void *mqd, 169 uint32_t pipe_id, uint32_t queue_id, 170 struct queue_properties *p, struct mm_struct *mms); 171 172 int kfd_destroy_mqd_cp(struct mqd_manager *mm, void *mqd, 173 enum kfd_preempt_type type, unsigned int timeout, 174 uint32_t pipe_id, uint32_t queue_id); 175 176 void kfd_free_mqd_cp(struct mqd_manager *mm, void *mqd, 177 struct kfd_mem_obj *mqd_mem_obj); 178 179 bool kfd_is_occupied_cp(struct mqd_manager *mm, void *mqd, 180 uint64_t queue_address, uint32_t pipe_id, 181 uint32_t queue_id); 182 183 int kfd_load_mqd_sdma(struct mqd_manager *mm, void *mqd, 184 uint32_t pipe_id, uint32_t queue_id, 185 struct queue_properties *p, struct mm_struct *mms); 186 187 int kfd_destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, 188 enum kfd_preempt_type type, unsigned int timeout, 189 uint32_t pipe_id, uint32_t queue_id); 190 191 bool kfd_is_occupied_sdma(struct mqd_manager *mm, void *mqd, 192 uint64_t queue_address, uint32_t pipe_id, 193 uint32_t queue_id); 194 195 void kfd_get_hiq_xcc_mqd(struct kfd_node *dev, 196 struct kfd_mem_obj *mqd_mem_obj, uint32_t virtual_xcc_id); 197 198 uint64_t kfd_hiq_mqd_stride(struct kfd_node *dev); 199 uint64_t kfd_mqd_stride(struct mqd_manager *mm, 200 struct queue_properties *q); 201 bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id, 202 uint32_t inst); 203 #endif /* KFD_MQD_MANAGER_H_ */ 204