1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #ifndef INTEL_RING_H 7 #define INTEL_RING_H 8 9 #include "i915_gem.h" /* GEM_BUG_ON */ 10 #include "i915_request.h" 11 #include "intel_ring_types.h" 12 13 struct intel_engine_cs; 14 15 struct intel_ring * 16 intel_engine_create_ring(struct intel_engine_cs *engine, int size); 17 18 u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords); 19 int intel_ring_cacheline_align(struct i915_request *rq); 20 21 unsigned int intel_ring_update_space(struct intel_ring *ring); 22 23 void __intel_ring_pin(struct intel_ring *ring); 24 int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww); 25 void intel_ring_unpin(struct intel_ring *ring); 26 void intel_ring_reset(struct intel_ring *ring, u32 tail); 27 28 void intel_ring_free(struct kref *ref); 29 intel_ring_get(struct intel_ring * ring)30 static inline struct intel_ring *intel_ring_get(struct intel_ring *ring) 31 { 32 kref_get(&ring->ref); 33 return ring; 34 } 35 intel_ring_put(struct intel_ring * ring)36 static inline void intel_ring_put(struct intel_ring *ring) 37 { 38 kref_put(&ring->ref, intel_ring_free); 39 } 40 intel_ring_advance(struct i915_request * rq,u32 * cs)41 static inline void intel_ring_advance(struct i915_request *rq, u32 *cs) 42 { 43 /* Dummy function. 44 * 45 * This serves as a placeholder in the code so that the reader 46 * can compare against the preceding intel_ring_begin() and 47 * check that the number of dwords emitted matches the space 48 * reserved for the command packet (i.e. the value passed to 49 * intel_ring_begin()). 50 */ 51 GEM_BUG_ON((rq->ring->vaddr + rq->ring->emit) != cs); 52 GEM_BUG_ON(!IS_ALIGNED(rq->ring->emit, 8)); /* RING_TAIL qword align */ 53 } 54 intel_ring_wrap(const struct intel_ring * ring,u32 pos)55 static inline u32 intel_ring_wrap(const struct intel_ring *ring, u32 pos) 56 { 57 return pos & (ring->size - 1); 58 } 59 intel_ring_direction(const struct intel_ring * ring,u32 next,u32 prev)60 static inline int intel_ring_direction(const struct intel_ring *ring, 61 u32 next, u32 prev) 62 { 63 typecheck(typeof(ring->size), next); 64 typecheck(typeof(ring->size), prev); 65 return (next - prev) << ring->wrap; 66 } 67 68 static inline bool intel_ring_offset_valid(const struct intel_ring * ring,unsigned int pos)69 intel_ring_offset_valid(const struct intel_ring *ring, 70 unsigned int pos) 71 { 72 if (pos & -ring->size) /* must be strictly within the ring */ 73 return false; 74 75 if (!IS_ALIGNED(pos, 8)) /* must be qword aligned */ 76 return false; 77 78 return true; 79 } 80 intel_ring_offset(const struct i915_request * rq,void * addr)81 static inline u32 intel_ring_offset(const struct i915_request *rq, void *addr) 82 { 83 /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */ 84 u32 offset = addr - rq->ring->vaddr; 85 86 GEM_BUG_ON(offset > rq->ring->size); 87 return intel_ring_wrap(rq->ring, offset); 88 } 89 90 static inline void assert_ring_tail_valid(const struct intel_ring * ring,unsigned int tail)91 assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail) 92 { 93 unsigned int head = READ_ONCE(ring->head); 94 95 GEM_BUG_ON(!intel_ring_offset_valid(ring, tail)); 96 97 /* 98 * "Ring Buffer Use" 99 * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 100 * Gen3 BSpec "1c Memory Interface Functions" / 2.3.4.5 101 * Gen4+ BSpec "1c Memory Interface and Command Stream" / 5.3.4.5 102 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the 103 * same cacheline, the Head Pointer must not be greater than the Tail 104 * Pointer." 105 * 106 * We use ring->head as the last known location of the actual RING_HEAD, 107 * it may have advanced but in the worst case it is equally the same 108 * as ring->head and so we should never program RING_TAIL to advance 109 * into the same cacheline as ring->head. 110 */ 111 #define cacheline(a) round_down(a, CACHELINE_BYTES) 112 GEM_BUG_ON(cacheline(tail) == cacheline(head) && tail < head); 113 #undef cacheline 114 } 115 116 static inline unsigned int intel_ring_set_tail(struct intel_ring * ring,unsigned int tail)117 intel_ring_set_tail(struct intel_ring *ring, unsigned int tail) 118 { 119 /* Whilst writes to the tail are strictly order, there is no 120 * serialisation between readers and the writers. The tail may be 121 * read by i915_request_retire() just as it is being updated 122 * by execlists, as although the breadcrumb is complete, the context 123 * switch hasn't been seen. 124 */ 125 assert_ring_tail_valid(ring, tail); 126 ring->tail = tail; 127 return tail; 128 } 129 130 static inline unsigned int __intel_ring_space(unsigned int head,unsigned int tail,unsigned int size)131 __intel_ring_space(unsigned int head, unsigned int tail, unsigned int size) 132 { 133 /* 134 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the 135 * same cacheline, the Head Pointer must not be greater than the Tail 136 * Pointer." 137 */ 138 GEM_BUG_ON(!is_power_of_2(size)); 139 return (head - tail - CACHELINE_BYTES) & (size - 1); 140 } 141 142 #endif /* INTEL_RING_H */ 143