1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2018-2022 Marvell International Ltd.
4 *
5 * Support functions for managing command queues used for
6 * various hardware blocks.
7 */
8
9 #include <errno.h>
10 #include <log.h>
11 #include <time.h>
12 #include <linux/delay.h>
13
14 #include <mach/cvmx-regs.h>
15 #include <mach/cvmx-csr.h>
16 #include <mach/cvmx-bootmem.h>
17 #include <mach/octeon-model.h>
18 #include <mach/cvmx-fuse.h>
19 #include <mach/octeon-feature.h>
20 #include <mach/cvmx-qlm.h>
21 #include <mach/octeon_qlm.h>
22 #include <mach/cvmx-pcie.h>
23 #include <mach/cvmx-coremask.h>
24
25 #include <mach/cvmx-fpa.h>
26 #include <mach/cvmx-cmd-queue.h>
27
28 #include <mach/cvmx-agl-defs.h>
29 #include <mach/cvmx-bgxx-defs.h>
30 #include <mach/cvmx-ciu-defs.h>
31 #include <mach/cvmx-gmxx-defs.h>
32 #include <mach/cvmx-gserx-defs.h>
33 #include <mach/cvmx-ilk-defs.h>
34 #include <mach/cvmx-ipd-defs.h>
35 #include <mach/cvmx-pcsx-defs.h>
36 #include <mach/cvmx-pcsxx-defs.h>
37 #include <mach/cvmx-pki-defs.h>
38 #include <mach/cvmx-pko-defs.h>
39 #include <mach/cvmx-xcv-defs.h>
40
41 #include <mach/cvmx-hwpko.h>
42 #include <mach/cvmx-ilk.h>
43 #include <mach/cvmx-pki.h>
44 #include <mach/cvmx-pko3.h>
45 #include <mach/cvmx-pko3-queue.h>
46 #include <mach/cvmx-pko3-resources.h>
47
48 #include <mach/cvmx-helper.h>
49 #include <mach/cvmx-helper-board.h>
50 #include <mach/cvmx-helper-cfg.h>
51
52 #include <mach/cvmx-helper-bgx.h>
53 #include <mach/cvmx-helper-cfg.h>
54 #include <mach/cvmx-helper-util.h>
55 #include <mach/cvmx-helper-pki.h>
56
57 #include <mach/cvmx-helper-util.h>
58 #include <mach/cvmx-dpi-defs.h>
59 #include <mach/cvmx-npei-defs.h>
60 #include <mach/cvmx-pexp-defs.h>
61
62 /**
63 * This application uses this pointer to access the global queue
64 * state. It points to a bootmem named block.
65 */
66 __cvmx_cmd_queue_all_state_t *__cvmx_cmd_queue_state_ptrs[CVMX_MAX_NODES];
67
68 /**
69 * @INTERNAL
70 * Initialize the Global queue state pointer.
71 *
72 * @return CVMX_CMD_QUEUE_SUCCESS or a failure code
73 */
__cvmx_cmd_queue_init_state_ptr(unsigned int node)74 cvmx_cmd_queue_result_t __cvmx_cmd_queue_init_state_ptr(unsigned int node)
75 {
76 const char *alloc_name = "cvmx_cmd_queues\0\0";
77 char s[4] = "_0";
78 const struct cvmx_bootmem_named_block_desc *block_desc = NULL;
79 unsigned int size;
80 u64 paddr_min = 0, paddr_max = 0;
81 void *ptr;
82
83 if (cvmx_likely(__cvmx_cmd_queue_state_ptrs[node]))
84 return CVMX_CMD_QUEUE_SUCCESS;
85
86 /* Add node# to block name */
87 if (node > 0) {
88 s[1] += node;
89 strcat((char *)alloc_name, s);
90 }
91
92 /* Find the named block in case it has been created already */
93 block_desc = cvmx_bootmem_find_named_block(alloc_name);
94 if (block_desc) {
95 __cvmx_cmd_queue_state_ptrs[node] =
96 (__cvmx_cmd_queue_all_state_t *)cvmx_phys_to_ptr(
97 block_desc->base_addr);
98 return CVMX_CMD_QUEUE_SUCCESS;
99 }
100
101 size = sizeof(*__cvmx_cmd_queue_state_ptrs[node]);
102
103 /* Rest f the code is to allocate a new named block */
104
105 /* Atomically allocate named block once, and zero it by default */
106 ptr = cvmx_bootmem_alloc_named_range_once(size, paddr_min, paddr_max,
107 128, alloc_name, NULL);
108
109 if (ptr) {
110 __cvmx_cmd_queue_state_ptrs[node] =
111 (__cvmx_cmd_queue_all_state_t *)ptr;
112 } else {
113 debug("ERROR: %s: Unable to get named block %s.\n", __func__,
114 alloc_name);
115 return CVMX_CMD_QUEUE_NO_MEMORY;
116 }
117 return CVMX_CMD_QUEUE_SUCCESS;
118 }
119
120 /**
121 * Initialize a command queue for use. The initial FPA buffer is
122 * allocated and the hardware unit is configured to point to the
123 * new command queue.
124 *
125 * @param queue_id Hardware command queue to initialize.
126 * @param max_depth Maximum outstanding commands that can be queued.
127 * @param fpa_pool FPA pool the command queues should come from.
128 * @param pool_size Size of each buffer in the FPA pool (bytes)
129 *
130 * @return CVMX_CMD_QUEUE_SUCCESS or a failure code
131 */
cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id,int max_depth,int fpa_pool,int pool_size)132 cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id,
133 int max_depth, int fpa_pool,
134 int pool_size)
135 {
136 __cvmx_cmd_queue_state_t *qstate;
137 cvmx_cmd_queue_result_t result;
138 unsigned int node;
139 unsigned int index;
140 int fpa_pool_min, fpa_pool_max;
141 union cvmx_fpa_ctl_status status;
142 void *buffer;
143
144 node = __cvmx_cmd_queue_get_node(queue_id);
145
146 index = __cvmx_cmd_queue_get_index(queue_id);
147 if (index >= NUM_ELEMENTS(__cvmx_cmd_queue_state_ptrs[node]->state)) {
148 printf("ERROR: %s: queue %#x out of range\n", __func__,
149 queue_id);
150 return CVMX_CMD_QUEUE_INVALID_PARAM;
151 }
152
153 result = __cvmx_cmd_queue_init_state_ptr(node);
154 if (result != CVMX_CMD_QUEUE_SUCCESS)
155 return result;
156
157 qstate = __cvmx_cmd_queue_get_state(queue_id);
158 if (!qstate)
159 return CVMX_CMD_QUEUE_INVALID_PARAM;
160
161 /*
162 * We artificially limit max_depth to 1<<20 words. It is an
163 * arbitrary limit.
164 */
165 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH) {
166 if (max_depth < 0 || max_depth > 1 << 20)
167 return CVMX_CMD_QUEUE_INVALID_PARAM;
168 } else if (max_depth != 0) {
169 return CVMX_CMD_QUEUE_INVALID_PARAM;
170 }
171
172 /* CVMX_FPA_NUM_POOLS maps to cvmx_fpa3_num_auras for FPA3 */
173 fpa_pool_min = node << 10;
174 fpa_pool_max = fpa_pool_min + CVMX_FPA_NUM_POOLS;
175
176 if (fpa_pool < fpa_pool_min || fpa_pool >= fpa_pool_max)
177 return CVMX_CMD_QUEUE_INVALID_PARAM;
178
179 if (pool_size < 128 || pool_size > (1 << 17))
180 return CVMX_CMD_QUEUE_INVALID_PARAM;
181
182 if (pool_size & 3)
183 debug("WARNING: %s: pool_size %d not multiple of 8\n", __func__,
184 pool_size);
185
186 /* See if someone else has already initialized the queue */
187 if (qstate->base_paddr) {
188 int depth;
189 static const char emsg[] = /* Common error message part */
190 "Queue already initialized with different ";
191
192 depth = (max_depth + qstate->pool_size_m1 - 1) /
193 qstate->pool_size_m1;
194 if (depth != qstate->max_depth) {
195 depth = qstate->max_depth * qstate->pool_size_m1;
196 debug("ERROR: %s: %s max_depth (%d).\n", __func__, emsg,
197 depth);
198 return CVMX_CMD_QUEUE_INVALID_PARAM;
199 }
200 if (fpa_pool != qstate->fpa_pool) {
201 debug("ERROR: %s: %s FPA pool (%d).\n", __func__, emsg,
202 (int)qstate->fpa_pool);
203 return CVMX_CMD_QUEUE_INVALID_PARAM;
204 }
205 if ((pool_size >> 3) - 1 != qstate->pool_size_m1) {
206 debug("ERROR: %s: %s FPA pool size (%u).\n", __func__,
207 emsg, (qstate->pool_size_m1 + 1) << 3);
208 return CVMX_CMD_QUEUE_INVALID_PARAM;
209 }
210 return CVMX_CMD_QUEUE_ALREADY_SETUP;
211 }
212
213 if (!(octeon_has_feature(OCTEON_FEATURE_FPA3))) {
214 status.u64 = csr_rd(CVMX_FPA_CTL_STATUS);
215 if (!status.s.enb) {
216 debug("ERROR: %s: FPA is not enabled.\n",
217 __func__);
218 return CVMX_CMD_QUEUE_NO_MEMORY;
219 }
220 }
221 buffer = cvmx_fpa_alloc(fpa_pool);
222 if (!buffer) {
223 debug("ERROR: %s: allocating first buffer.\n", __func__);
224 return CVMX_CMD_QUEUE_NO_MEMORY;
225 }
226
227 index = (pool_size >> 3) - 1;
228 qstate->pool_size_m1 = index;
229 qstate->max_depth = (max_depth + index - 1) / index;
230 qstate->index = 0;
231 qstate->fpa_pool = fpa_pool;
232 qstate->base_paddr = cvmx_ptr_to_phys(buffer);
233
234 /* Initialize lock */
235 __cvmx_cmd_queue_lock_init(queue_id);
236 return CVMX_CMD_QUEUE_SUCCESS;
237 }
238
239 /**
240 * Return the command buffer to be written to. The purpose of this
241 * function is to allow CVMX routine access to the low level buffer
242 * for initial hardware setup. User applications should not call this
243 * function directly.
244 *
245 * @param queue_id Command queue to query
246 *
247 * @return Command buffer or NULL on failure
248 */
cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id)249 void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id)
250 {
251 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
252
253 if (qptr && qptr->base_paddr)
254 return cvmx_phys_to_ptr((u64)qptr->base_paddr);
255 else
256 return NULL;
257 }
258
__cvmx_cmd_queue_add_blk(__cvmx_cmd_queue_state_t * qptr)259 static u64 *__cvmx_cmd_queue_add_blk(__cvmx_cmd_queue_state_t *qptr)
260 {
261 u64 *cmd_ptr;
262 u64 *new_buffer;
263 u64 new_paddr;
264
265 /* Get base vaddr of current (full) block */
266 cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
267
268 /* Allocate a new block from the per-queue pool */
269 new_buffer = (u64 *)cvmx_fpa_alloc(qptr->fpa_pool);
270
271 /* Check for allocation failure */
272 if (cvmx_unlikely(!new_buffer))
273 return NULL;
274
275 /* Zero out the new block link pointer,
276 * in case this block will be filled to the rim
277 */
278 new_buffer[qptr->pool_size_m1] = ~0ull;
279
280 /* Get physical address of the new buffer */
281 new_paddr = cvmx_ptr_to_phys(new_buffer);
282
283 /* Store the physical link address at the end of current full block */
284 cmd_ptr[qptr->pool_size_m1] = new_paddr;
285
286 /* Store the physical address in the queue state structure */
287 qptr->base_paddr = new_paddr;
288 qptr->index = 0;
289
290 /* Return the virtual base of the new block */
291 return new_buffer;
292 }
293
294 /**
295 * @INTERNAL
296 * Add command words into a queue, handles all the corener cases
297 * where only some of the words might fit into the current block,
298 * and a new block may need to be allocated.
299 * Locking and argument checks are done in the front-end in-line
300 * functions that call this one for the rare corner cases.
301 */
302 cvmx_cmd_queue_result_t
__cvmx_cmd_queue_write_raw(cvmx_cmd_queue_id_t queue_id,__cvmx_cmd_queue_state_t * qptr,int cmd_count,const u64 * cmds)303 __cvmx_cmd_queue_write_raw(cvmx_cmd_queue_id_t queue_id,
304 __cvmx_cmd_queue_state_t *qptr, int cmd_count,
305 const u64 *cmds)
306 {
307 u64 *cmd_ptr;
308 unsigned int index;
309
310 cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
311 index = qptr->index;
312
313 /* Enforce queue depth limit, if enabled, once per block */
314 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && cvmx_unlikely(qptr->max_depth)) {
315 unsigned int depth = cvmx_cmd_queue_length(queue_id);
316
317 depth /= qptr->pool_size_m1;
318
319 if (cvmx_unlikely(depth > qptr->max_depth))
320 return CVMX_CMD_QUEUE_FULL;
321 }
322
323 /*
324 * If the block allocation fails, even the words that we wrote
325 * to the current block will not count because the 'index' will
326 * not be comitted.
327 * The loop is run 'count + 1' times to take care of the tail
328 * case, where the buffer is full to the rim, so the link
329 * pointer must be filled with a valid address.
330 */
331 while (cmd_count >= 0) {
332 if (index >= qptr->pool_size_m1) {
333 /* Block is full, get another one and proceed */
334 cmd_ptr = __cvmx_cmd_queue_add_blk(qptr);
335
336 /* Baul on allocation error w/o comitting anything */
337 if (cvmx_unlikely(!cmd_ptr))
338 return CVMX_CMD_QUEUE_NO_MEMORY;
339
340 /* Reset index for start of new block */
341 index = 0;
342 }
343 /* Exit Loop on 'count + 1' iterations */
344 if (cmd_count <= 0)
345 break;
346 /* Store commands into queue block while there is space */
347 cmd_ptr[index++] = *cmds++;
348 cmd_count--;
349 } /* while cmd_count */
350
351 /* Commit added words if all is well */
352 qptr->index = index;
353
354 return CVMX_CMD_QUEUE_SUCCESS;
355 }
356