1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * dma-fence-util: misc functions for dma_fence objects
4 *
5 * Copyright (C) 2022 Advanced Micro Devices, Inc.
6 * Authors:
7 * Christian König <christian.koenig@amd.com>
8 */
9
10 #include <linux/dma-fence.h>
11 #include <linux/dma-fence-array.h>
12 #include <linux/dma-fence-chain.h>
13 #include <linux/dma-fence-unwrap.h>
14 #include <linux/slab.h>
15 #include <linux/sort.h>
16
17 /* Internal helper to start new array iteration, don't use directly */
18 static struct dma_fence *
__dma_fence_unwrap_array(struct dma_fence_unwrap * cursor)19 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
20 {
21 cursor->array = dma_fence_chain_contained(cursor->chain);
22 cursor->index = 0;
23 return dma_fence_array_first(cursor->array);
24 }
25
26 /**
27 * dma_fence_unwrap_first - return the first fence from fence containers
28 * @head: the entrypoint into the containers
29 * @cursor: current position inside the containers
30 *
31 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
32 * first fence.
33 */
dma_fence_unwrap_first(struct dma_fence * head,struct dma_fence_unwrap * cursor)34 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
35 struct dma_fence_unwrap *cursor)
36 {
37 cursor->chain = dma_fence_get(head);
38 return __dma_fence_unwrap_array(cursor);
39 }
40 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
41
42 /**
43 * dma_fence_unwrap_next - return the next fence from a fence containers
44 * @cursor: current position inside the containers
45 *
46 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
47 * the next fence from them.
48 */
dma_fence_unwrap_next(struct dma_fence_unwrap * cursor)49 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
50 {
51 struct dma_fence *tmp;
52
53 ++cursor->index;
54 tmp = dma_fence_array_next(cursor->array, cursor->index);
55 if (tmp)
56 return tmp;
57
58 cursor->chain = dma_fence_chain_walk(cursor->chain);
59 return __dma_fence_unwrap_array(cursor);
60 }
61 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
62
63
fence_cmp(const void * _a,const void * _b)64 static int fence_cmp(const void *_a, const void *_b)
65 {
66 struct dma_fence *a = *(struct dma_fence **)_a;
67 struct dma_fence *b = *(struct dma_fence **)_b;
68
69 if (a->context < b->context)
70 return -1;
71 else if (a->context > b->context)
72 return 1;
73
74 if (dma_fence_is_later(b, a))
75 return 1;
76 else if (dma_fence_is_later(a, b))
77 return -1;
78
79 return 0;
80 }
81
82 /**
83 * dma_fence_dedup_array - Sort and deduplicate an array of dma_fence pointers
84 * @fences: Array of dma_fence pointers to be deduplicated
85 * @num_fences: Number of entries in the @fences array
86 *
87 * Sorts the input array by context, then removes duplicate
88 * fences with the same context, keeping only the most recent one.
89 *
90 * The array is modified in-place and unreferenced duplicate fences are released
91 * via dma_fence_put(). The function returns the new number of fences after
92 * deduplication.
93 *
94 * Return: Number of unique fences remaining in the array.
95 */
dma_fence_dedup_array(struct dma_fence ** fences,int num_fences)96 int dma_fence_dedup_array(struct dma_fence **fences, int num_fences)
97 {
98 int i, j;
99
100 sort(fences, num_fences, sizeof(*fences), fence_cmp, NULL);
101
102 /*
103 * Only keep the most recent fence for each context.
104 */
105 j = 0;
106 for (i = 1; i < num_fences; i++) {
107 if (fences[i]->context == fences[j]->context)
108 dma_fence_put(fences[i]);
109 else
110 fences[++j] = fences[i];
111 }
112
113 return ++j;
114 }
115 EXPORT_SYMBOL_GPL(dma_fence_dedup_array);
116
117 /* Implementation for the dma_fence_merge() marco, don't use directly */
__dma_fence_unwrap_merge(unsigned int num_fences,struct dma_fence ** fences,struct dma_fence_unwrap * iter)118 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
119 struct dma_fence **fences,
120 struct dma_fence_unwrap *iter)
121 {
122 struct dma_fence *tmp, *unsignaled = NULL, **array;
123 struct dma_fence_array *result;
124 ktime_t timestamp;
125 int i, count;
126
127 count = 0;
128 timestamp = ns_to_ktime(0);
129 for (i = 0; i < num_fences; ++i) {
130 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
131 if (!dma_fence_is_signaled(tmp)) {
132 dma_fence_put(unsignaled);
133 unsignaled = dma_fence_get(tmp);
134 ++count;
135 } else {
136 ktime_t t = dma_fence_timestamp(tmp);
137
138 if (ktime_after(t, timestamp))
139 timestamp = t;
140 }
141 }
142 }
143
144 /*
145 * If we couldn't find a pending fence just return a private signaled
146 * fence with the timestamp of the last signaled one.
147 *
148 * Or if there was a single unsignaled fence left we can return it
149 * directly and early since that is a major path on many workloads.
150 */
151 if (count == 0)
152 return dma_fence_allocate_private_stub(timestamp);
153 else if (count == 1)
154 return unsignaled;
155
156 dma_fence_put(unsignaled);
157
158 array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
159 if (!array)
160 return NULL;
161
162 count = 0;
163 for (i = 0; i < num_fences; ++i) {
164 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
165 if (!dma_fence_is_signaled(tmp)) {
166 array[count++] = dma_fence_get(tmp);
167 } else {
168 ktime_t t = dma_fence_timestamp(tmp);
169
170 if (ktime_after(t, timestamp))
171 timestamp = t;
172 }
173 }
174 }
175
176 if (count == 0 || count == 1)
177 goto return_fastpath;
178
179 count = dma_fence_dedup_array(array, count);
180
181 if (count > 1) {
182 result = dma_fence_array_create(count, array,
183 dma_fence_context_alloc(1),
184 1, false);
185 if (!result) {
186 for (i = 0; i < count; i++)
187 dma_fence_put(array[i]);
188 tmp = NULL;
189 goto return_tmp;
190 }
191 return &result->base;
192 }
193
194 return_fastpath:
195 if (count == 0)
196 tmp = dma_fence_allocate_private_stub(timestamp);
197 else
198 tmp = array[0];
199
200 return_tmp:
201 kfree(array);
202 return tmp;
203 }
204 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);
205