1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <linux/dma-fence.h>
7 #include <linux/interval_tree_generic.h>
8 #include <linux/slab.h>
9
10 #include "xe_macros.h"
11 #include "xe_range_fence.h"
12
13 #define XE_RANGE_TREE_START(_node) ((_node)->start)
14 #define XE_RANGE_TREE_LAST(_node) ((_node)->last)
15
16 INTERVAL_TREE_DEFINE(struct xe_range_fence, rb, u64, __subtree_last,
17 XE_RANGE_TREE_START, XE_RANGE_TREE_LAST, static,
18 xe_range_fence_tree);
19
20 static void
xe_range_fence_signal_notify(struct dma_fence * fence,struct dma_fence_cb * cb)21 xe_range_fence_signal_notify(struct dma_fence *fence, struct dma_fence_cb *cb)
22 {
23 struct xe_range_fence *rfence = container_of(cb, typeof(*rfence), cb);
24 struct xe_range_fence_tree *tree = rfence->tree;
25
26 llist_add(&rfence->link, &tree->list);
27 }
28
__xe_range_fence_tree_cleanup(struct xe_range_fence_tree * tree)29 static bool __xe_range_fence_tree_cleanup(struct xe_range_fence_tree *tree)
30 {
31 struct llist_node *node = llist_del_all(&tree->list);
32 struct xe_range_fence *rfence, *next;
33
34 llist_for_each_entry_safe(rfence, next, node, link) {
35 xe_range_fence_tree_remove(rfence, &tree->root);
36 dma_fence_put(rfence->fence);
37 kfree(rfence);
38 }
39
40 return !!node;
41 }
42
43 /**
44 * xe_range_fence_insert() - range fence insert
45 * @tree: range fence tree to insert intoi
46 * @rfence: range fence
47 * @ops: range fence ops
48 * @start: start address of range fence
49 * @last: last address of range fence
50 * @fence: dma fence which signals range fence can be removed + freed
51 *
52 * Return: 0 on success, non-zero on failure
53 */
xe_range_fence_insert(struct xe_range_fence_tree * tree,struct xe_range_fence * rfence,const struct xe_range_fence_ops * ops,u64 start,u64 last,struct dma_fence * fence)54 int xe_range_fence_insert(struct xe_range_fence_tree *tree,
55 struct xe_range_fence *rfence,
56 const struct xe_range_fence_ops *ops,
57 u64 start, u64 last, struct dma_fence *fence)
58 {
59 int err = 0;
60
61 __xe_range_fence_tree_cleanup(tree);
62
63 if (dma_fence_is_signaled(fence))
64 goto free;
65
66 rfence->ops = ops;
67 rfence->start = start;
68 rfence->last = last;
69 rfence->tree = tree;
70 rfence->fence = dma_fence_get(fence);
71 err = dma_fence_add_callback(fence, &rfence->cb,
72 xe_range_fence_signal_notify);
73 if (err == -ENOENT) {
74 dma_fence_put(fence);
75 err = 0;
76 goto free;
77 } else if (err == 0) {
78 xe_range_fence_tree_insert(rfence, &tree->root);
79 return 0;
80 }
81
82 free:
83 if (ops->free)
84 ops->free(rfence);
85
86 return err;
87 }
88
xe_range_fence_tree_remove_all(struct xe_range_fence_tree * tree)89 static void xe_range_fence_tree_remove_all(struct xe_range_fence_tree *tree)
90 {
91 struct xe_range_fence *rfence;
92 bool retry = true;
93
94 rfence = xe_range_fence_tree_iter_first(&tree->root, 0, U64_MAX);
95 while (rfence) {
96 /* Should be ok with the minimalistic callback */
97 if (dma_fence_remove_callback(rfence->fence, &rfence->cb))
98 llist_add(&rfence->link, &tree->list);
99 rfence = xe_range_fence_tree_iter_next(rfence, 0, U64_MAX);
100 }
101
102 while (retry)
103 retry = __xe_range_fence_tree_cleanup(tree);
104 }
105
106 /**
107 * xe_range_fence_tree_init() - Init range fence tree
108 * @tree: range fence tree
109 */
xe_range_fence_tree_init(struct xe_range_fence_tree * tree)110 void xe_range_fence_tree_init(struct xe_range_fence_tree *tree)
111 {
112 memset(tree, 0, sizeof(*tree));
113 }
114
115 /**
116 * xe_range_fence_tree_fini() - Fini range fence tree
117 * @tree: range fence tree
118 */
xe_range_fence_tree_fini(struct xe_range_fence_tree * tree)119 void xe_range_fence_tree_fini(struct xe_range_fence_tree *tree)
120 {
121 xe_range_fence_tree_remove_all(tree);
122 XE_WARN_ON(!RB_EMPTY_ROOT(&tree->root.rb_root));
123 }
124
125 /**
126 * xe_range_fence_tree_first() - range fence tree iterator first
127 * @tree: range fence tree
128 * @start: start address of range fence
129 * @last: last address of range fence
130 *
131 * Return: first range fence found in range or NULL
132 */
133 struct xe_range_fence *
xe_range_fence_tree_first(struct xe_range_fence_tree * tree,u64 start,u64 last)134 xe_range_fence_tree_first(struct xe_range_fence_tree *tree, u64 start,
135 u64 last)
136 {
137 return xe_range_fence_tree_iter_first(&tree->root, start, last);
138 }
139
140 /**
141 * xe_range_fence_tree_next() - range fence tree iterator next
142 * @rfence: current range fence
143 * @start: start address of range fence
144 * @last: last address of range fence
145 *
146 * Return: next range fence found in range or NULL
147 */
148 struct xe_range_fence *
xe_range_fence_tree_next(struct xe_range_fence * rfence,u64 start,u64 last)149 xe_range_fence_tree_next(struct xe_range_fence *rfence, u64 start, u64 last)
150 {
151 return xe_range_fence_tree_iter_next(rfence, start, last);
152 }
153
xe_range_fence_free(struct xe_range_fence * rfence)154 static void xe_range_fence_free(struct xe_range_fence *rfence)
155 {
156 kfree(rfence);
157 }
158
159 const struct xe_range_fence_ops xe_range_fence_kfree_ops = {
160 .free = xe_range_fence_free,
161 };
162