1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #ifndef _TTM_RESOURCE_H_ 26 #define _TTM_RESOURCE_H_ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/iosys-map.h> 32 #include <linux/dma-fence.h> 33 34 #include <drm/drm_print.h> 35 #include <drm/ttm/ttm_caching.h> 36 #include <drm/ttm/ttm_kmap_iter.h> 37 38 #define TTM_MAX_BO_PRIORITY 4U 39 #define TTM_NUM_MEM_TYPES 8 40 41 struct ttm_device; 42 struct ttm_resource_manager; 43 struct ttm_resource; 44 struct ttm_place; 45 struct ttm_buffer_object; 46 struct ttm_placement; 47 struct iosys_map; 48 struct io_mapping; 49 struct sg_table; 50 struct scatterlist; 51 52 /** 53 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses 54 */ 55 enum ttm_lru_item_type { 56 /** @TTM_LRU_RESOURCE: The resource subclass */ 57 TTM_LRU_RESOURCE, 58 /** @TTM_LRU_HITCH: The iterator hitch subclass */ 59 TTM_LRU_HITCH 60 }; 61 62 /** 63 * struct ttm_lru_item - The TTM lru list node base class 64 * @link: The list link 65 * @type: The subclass type 66 */ 67 struct ttm_lru_item { 68 struct list_head link; 69 enum ttm_lru_item_type type; 70 }; 71 72 /** 73 * ttm_lru_item_init() - initialize a struct ttm_lru_item 74 * @item: The item to initialize 75 * @type: The subclass type 76 */ ttm_lru_item_init(struct ttm_lru_item * item,enum ttm_lru_item_type type)77 static inline void ttm_lru_item_init(struct ttm_lru_item *item, 78 enum ttm_lru_item_type type) 79 { 80 item->type = type; 81 INIT_LIST_HEAD(&item->link); 82 } 83 ttm_lru_item_is_res(const struct ttm_lru_item * item)84 static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item) 85 { 86 return item->type == TTM_LRU_RESOURCE; 87 } 88 89 struct ttm_resource_manager_func { 90 /** 91 * struct ttm_resource_manager_func member alloc 92 * 93 * @man: Pointer to a memory type manager. 94 * @bo: Pointer to the buffer object we're allocating space for. 95 * @place: Placement details. 96 * @res: Resulting pointer to the ttm_resource. 97 * 98 * This function should allocate space in the memory type managed 99 * by @man. Placement details if applicable are given by @place. If 100 * successful, a filled in ttm_resource object should be returned in 101 * @res. @res::start should be set to a value identifying the beginning 102 * of the range allocated, and the function should return zero. 103 * If the manager can't fulfill the request -ENOSPC should be returned. 104 * If a system error occurred, preventing the request to be fulfilled, 105 * the function should return a negative error code. 106 * 107 * This function may not be called from within atomic context and needs 108 * to take care of its own locking to protect any data structures 109 * managing the space. 110 */ 111 int (*alloc)(struct ttm_resource_manager *man, 112 struct ttm_buffer_object *bo, 113 const struct ttm_place *place, 114 struct ttm_resource **res); 115 116 /** 117 * struct ttm_resource_manager_func member free 118 * 119 * @man: Pointer to a memory type manager. 120 * @res: Pointer to a struct ttm_resource to be freed. 121 * 122 * This function frees memory type resources previously allocated. 123 * May not be called from within atomic context. 124 */ 125 void (*free)(struct ttm_resource_manager *man, 126 struct ttm_resource *res); 127 128 /** 129 * struct ttm_resource_manager_func member intersects 130 * 131 * @man: Pointer to a memory type manager. 132 * @res: Pointer to a struct ttm_resource to be checked. 133 * @place: Placement to check against. 134 * @size: Size of the check. 135 * 136 * Test if @res intersects with @place + @size. Used to judge if 137 * evictions are valueable or not. 138 */ 139 bool (*intersects)(struct ttm_resource_manager *man, 140 struct ttm_resource *res, 141 const struct ttm_place *place, 142 size_t size); 143 144 /** 145 * struct ttm_resource_manager_func member compatible 146 * 147 * @man: Pointer to a memory type manager. 148 * @res: Pointer to a struct ttm_resource to be checked. 149 * @place: Placement to check against. 150 * @size: Size of the check. 151 * 152 * Test if @res compatible with @place + @size. Used to check of 153 * the need to move the backing store or not. 154 */ 155 bool (*compatible)(struct ttm_resource_manager *man, 156 struct ttm_resource *res, 157 const struct ttm_place *place, 158 size_t size); 159 160 /** 161 * struct ttm_resource_manager_func member debug 162 * 163 * @man: Pointer to a memory type manager. 164 * @printer: Prefix to be used in printout to identify the caller. 165 * 166 * This function is called to print out the state of the memory 167 * type manager to aid debugging of out-of-memory conditions. 168 * It may not be called from within atomic context. 169 */ 170 void (*debug)(struct ttm_resource_manager *man, 171 struct drm_printer *printer); 172 }; 173 174 /** 175 * struct ttm_resource_manager 176 * 177 * @use_type: The memory type is enabled. 178 * @use_tt: If a TT object should be used for the backing store. 179 * @size: Size of the managed region. 180 * @bdev: ttm device this manager belongs to 181 * @func: structure pointer implementing the range manager. See above 182 * @move_lock: lock for move fence 183 * @move: The fence of the last pipelined move operation. 184 * @lru: The lru list for this memory type. 185 * 186 * This structure is used to identify and manage memory types for a device. 187 */ 188 struct ttm_resource_manager { 189 /* 190 * No protection. Constant from start. 191 */ 192 bool use_type; 193 bool use_tt; 194 struct ttm_device *bdev; 195 uint64_t size; 196 const struct ttm_resource_manager_func *func; 197 spinlock_t move_lock; 198 199 /* 200 * Protected by @move_lock. 201 */ 202 struct dma_fence *move; 203 204 /* 205 * Protected by the bdev->lru_lock. 206 */ 207 struct list_head lru[TTM_MAX_BO_PRIORITY]; 208 209 /** 210 * @usage: How much of the resources are used, protected by the 211 * bdev->lru_lock. 212 */ 213 uint64_t usage; 214 }; 215 216 /** 217 * struct ttm_bus_placement 218 * 219 * @addr: mapped virtual address 220 * @offset: physical addr 221 * @is_iomem: is this io memory ? 222 * @caching: See enum ttm_caching 223 * 224 * Structure indicating the bus placement of an object. 225 */ 226 struct ttm_bus_placement { 227 void *addr; 228 phys_addr_t offset; 229 bool is_iomem; 230 enum ttm_caching caching; 231 }; 232 233 /** 234 * struct ttm_resource 235 * 236 * @start: Start of the allocation. 237 * @size: Actual size of resource in bytes. 238 * @mem_type: Resource type of the allocation. 239 * @placement: Placement flags. 240 * @bus: Placement on io bus accessible to the CPU 241 * @bo: weak reference to the BO, protected by ttm_device::lru_lock 242 * 243 * Structure indicating the placement and space resources used by a 244 * buffer object. 245 */ 246 struct ttm_resource { 247 unsigned long start; 248 size_t size; 249 uint32_t mem_type; 250 uint32_t placement; 251 struct ttm_bus_placement bus; 252 struct ttm_buffer_object *bo; 253 254 /** 255 * @lru: Least recently used list, see &ttm_resource_manager.lru 256 */ 257 struct ttm_lru_item lru; 258 }; 259 260 /** 261 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource 262 * @item: The struct ttm_lru_item to downcast 263 * 264 * Return: Pointer to the embedding struct ttm_resource 265 */ 266 static inline struct ttm_resource * ttm_lru_item_to_res(struct ttm_lru_item * item)267 ttm_lru_item_to_res(struct ttm_lru_item *item) 268 { 269 return container_of(item, struct ttm_resource, lru); 270 } 271 272 /** 273 * struct ttm_lru_bulk_move_pos 274 * 275 * @first: first res in the bulk move range 276 * @last: last res in the bulk move range 277 * 278 * Range of resources for a lru bulk move. 279 */ 280 struct ttm_lru_bulk_move_pos { 281 struct ttm_resource *first; 282 struct ttm_resource *last; 283 }; 284 285 /** 286 * struct ttm_lru_bulk_move 287 * @pos: first/last lru entry for resources in the each domain/priority 288 * @cursor_list: The list of cursors currently traversing any of 289 * the sublists of @pos. Protected by the ttm device's lru_lock. 290 * 291 * Container for the current bulk move state. Should be used with 292 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). 293 * All BOs in a bulk_move structure need to share the same reservation object to 294 * ensure that the bulk as a whole is locked for eviction even if only one BO of 295 * the bulk is evicted. 296 */ 297 struct ttm_lru_bulk_move { 298 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; 299 struct list_head cursor_list; 300 }; 301 302 /** 303 * struct ttm_resource_cursor 304 * @man: The resource manager currently being iterated over 305 * @hitch: A hitch list node inserted before the next resource 306 * to iterate over. 307 * @bulk_link: A list link for the list of cursors traversing the 308 * bulk sublist of @bulk. Protected by the ttm device's lru_lock. 309 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is 310 * inserted to. NULL if none. Never dereference this pointer since 311 * the struct ttm_lru_bulk_move object pointed to might have been 312 * freed. The pointer is only for comparison. 313 * @mem_type: The memory type of the LRU list being traversed. 314 * This field is valid iff @bulk != NULL. 315 * @priority: the current priority 316 * 317 * Cursor to iterate over the resources in a manager. 318 */ 319 struct ttm_resource_cursor { 320 struct ttm_resource_manager *man; 321 struct ttm_lru_item hitch; 322 struct list_head bulk_link; 323 struct ttm_lru_bulk_move *bulk; 324 unsigned int mem_type; 325 unsigned int priority; 326 }; 327 328 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor); 329 330 /** 331 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + 332 * struct sg_table backed struct ttm_resource. 333 * @base: Embedded struct ttm_kmap_iter providing the usage interface. 334 * @iomap: struct io_mapping representing the underlying linear io_memory. 335 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. 336 * @start: Offset that needs to be subtracted from @st to make 337 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 338 * @cache: Scatterlist traversal cache for fast lookups. 339 * @cache.sg: Pointer to the currently cached scatterlist segment. 340 * @cache.i: First index of @sg. PAGE_SIZE granularity. 341 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. 342 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. 343 */ 344 struct ttm_kmap_iter_iomap { 345 struct ttm_kmap_iter base; 346 struct io_mapping *iomap; 347 struct sg_table *st; 348 resource_size_t start; 349 struct { 350 struct scatterlist *sg; 351 pgoff_t i; 352 pgoff_t end; 353 pgoff_t offs; 354 } cache; 355 }; 356 357 /** 358 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io 359 * @base: The base iterator 360 * @dmap: Points to the starting address of the region 361 * @needs_unmap: Whether we need to unmap on fini 362 */ 363 struct ttm_kmap_iter_linear_io { 364 struct ttm_kmap_iter base; 365 struct iosys_map dmap; 366 bool needs_unmap; 367 }; 368 369 /** 370 * ttm_resource_manager_set_used 371 * 372 * @man: A memory manager object. 373 * @used: usage state to set. 374 * 375 * Set the manager in use flag. If disabled the manager is no longer 376 * used for object placement. 377 */ 378 static inline void ttm_resource_manager_set_used(struct ttm_resource_manager * man,bool used)379 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) 380 { 381 int i; 382 383 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) 384 WARN_ON(!list_empty(&man->lru[i])); 385 man->use_type = used; 386 } 387 388 /** 389 * ttm_resource_manager_used 390 * 391 * @man: Manager to get used state for 392 * 393 * Get the in use flag for a manager. 394 * Returns: 395 * true is used, false if not. 396 */ ttm_resource_manager_used(struct ttm_resource_manager * man)397 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) 398 { 399 return man->use_type; 400 } 401 402 /** 403 * ttm_resource_manager_cleanup 404 * 405 * @man: A memory manager object. 406 * 407 * Cleanup the move fences from the memory manager object. 408 */ 409 static inline void ttm_resource_manager_cleanup(struct ttm_resource_manager * man)410 ttm_resource_manager_cleanup(struct ttm_resource_manager *man) 411 { 412 dma_fence_put(man->move); 413 man->move = NULL; 414 } 415 416 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); 417 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); 418 void ttm_lru_bulk_move_fini(struct ttm_device *bdev, 419 struct ttm_lru_bulk_move *bulk); 420 421 void ttm_resource_add_bulk_move(struct ttm_resource *res, 422 struct ttm_buffer_object *bo); 423 void ttm_resource_del_bulk_move(struct ttm_resource *res, 424 struct ttm_buffer_object *bo); 425 void ttm_resource_move_to_lru_tail(struct ttm_resource *res); 426 427 void ttm_resource_init(struct ttm_buffer_object *bo, 428 const struct ttm_place *place, 429 struct ttm_resource *res); 430 void ttm_resource_fini(struct ttm_resource_manager *man, 431 struct ttm_resource *res); 432 433 int ttm_resource_alloc(struct ttm_buffer_object *bo, 434 const struct ttm_place *place, 435 struct ttm_resource **res); 436 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); 437 bool ttm_resource_intersects(struct ttm_device *bdev, 438 struct ttm_resource *res, 439 const struct ttm_place *place, 440 size_t size); 441 bool ttm_resource_compatible(struct ttm_resource *res, 442 struct ttm_placement *placement, 443 bool evicting); 444 void ttm_resource_set_bo(struct ttm_resource *res, 445 struct ttm_buffer_object *bo); 446 447 void ttm_resource_manager_init(struct ttm_resource_manager *man, 448 struct ttm_device *bdev, 449 uint64_t size); 450 451 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 452 struct ttm_resource_manager *man); 453 454 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); 455 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 456 struct drm_printer *p); 457 458 struct ttm_resource * 459 ttm_resource_manager_first(struct ttm_resource_manager *man, 460 struct ttm_resource_cursor *cursor); 461 struct ttm_resource * 462 ttm_resource_manager_next(struct ttm_resource_cursor *cursor); 463 464 struct ttm_resource * 465 ttm_lru_first_res_or_null(struct list_head *head); 466 467 /** 468 * ttm_resource_manager_for_each_res - iterate over all resources 469 * @man: the resource manager 470 * @cursor: struct ttm_resource_cursor for the current position 471 * @res: the current resource 472 * 473 * Iterate over all the evictable resources in a resource manager. 474 */ 475 #define ttm_resource_manager_for_each_res(man, cursor, res) \ 476 for (res = ttm_resource_manager_first(man, cursor); res; \ 477 res = ttm_resource_manager_next(cursor)) 478 479 struct ttm_kmap_iter * 480 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 481 struct io_mapping *iomap, 482 struct sg_table *st, 483 resource_size_t start); 484 485 struct ttm_kmap_iter_linear_io; 486 487 struct ttm_kmap_iter * 488 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 489 struct ttm_device *bdev, 490 struct ttm_resource *mem); 491 492 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 493 struct ttm_device *bdev, 494 struct ttm_resource *mem); 495 496 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, 497 struct dentry * parent, 498 const char *name); 499 #endif 500