1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2019, Mellanox Technologies */
3
4 #ifndef _DR_TYPES_
5 #define _DR_TYPES_
6
7 #include <linux/mlx5/vport.h>
8 #include <linux/refcount.h>
9 #include "fs_core.h"
10 #include "wq.h"
11 #include "lib/mlx5.h"
12 #include "mlx5_ifc_dr.h"
13 #include "mlx5dr.h"
14 #include "dr_dbg.h"
15
16 #define DR_RULE_MAX_STES 18
17 #define DR_ACTION_MAX_STES 5
18 #define DR_STE_SVLAN 0x1
19 #define DR_STE_CVLAN 0x2
20 #define DR_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4)
21 #define DR_NUM_OF_FLEX_PARSERS 8
22 #define DR_STE_MAX_FLEX_0_ID 3
23 #define DR_STE_MAX_FLEX_1_ID 7
24
25 #define mlx5dr_err(dmn, arg...) mlx5_core_err((dmn)->mdev, ##arg)
26 #define mlx5dr_info(dmn, arg...) mlx5_core_info((dmn)->mdev, ##arg)
27 #define mlx5dr_dbg(dmn, arg...) mlx5_core_dbg((dmn)->mdev, ##arg)
28
dr_is_flex_parser_0_id(u8 parser_id)29 static inline bool dr_is_flex_parser_0_id(u8 parser_id)
30 {
31 return parser_id <= DR_STE_MAX_FLEX_0_ID;
32 }
33
dr_is_flex_parser_1_id(u8 parser_id)34 static inline bool dr_is_flex_parser_1_id(u8 parser_id)
35 {
36 return parser_id > DR_STE_MAX_FLEX_0_ID;
37 }
38
39 enum mlx5dr_icm_chunk_size {
40 DR_CHUNK_SIZE_1,
41 DR_CHUNK_SIZE_MIN = DR_CHUNK_SIZE_1, /* keep updated when changing */
42 DR_CHUNK_SIZE_2,
43 DR_CHUNK_SIZE_4,
44 DR_CHUNK_SIZE_8,
45 DR_CHUNK_SIZE_16,
46 DR_CHUNK_SIZE_32,
47 DR_CHUNK_SIZE_64,
48 DR_CHUNK_SIZE_128,
49 DR_CHUNK_SIZE_256,
50 DR_CHUNK_SIZE_512,
51 DR_CHUNK_SIZE_1K,
52 DR_CHUNK_SIZE_2K,
53 DR_CHUNK_SIZE_4K,
54 DR_CHUNK_SIZE_8K,
55 DR_CHUNK_SIZE_16K,
56 DR_CHUNK_SIZE_32K,
57 DR_CHUNK_SIZE_64K,
58 DR_CHUNK_SIZE_128K,
59 DR_CHUNK_SIZE_256K,
60 DR_CHUNK_SIZE_512K,
61 DR_CHUNK_SIZE_1024K,
62 DR_CHUNK_SIZE_2048K,
63 DR_CHUNK_SIZE_MAX,
64 };
65
66 enum mlx5dr_icm_type {
67 DR_ICM_TYPE_STE,
68 DR_ICM_TYPE_MODIFY_ACTION,
69 };
70
71 static inline enum mlx5dr_icm_chunk_size
mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)72 mlx5dr_icm_next_higher_chunk(enum mlx5dr_icm_chunk_size chunk)
73 {
74 chunk += 2;
75 if (chunk < DR_CHUNK_SIZE_MAX)
76 return chunk;
77
78 return DR_CHUNK_SIZE_MAX;
79 }
80
81 enum {
82 DR_STE_SIZE = 64,
83 DR_STE_SIZE_CTRL = 32,
84 DR_STE_SIZE_MATCH_TAG = 32,
85 DR_STE_SIZE_TAG = 16,
86 DR_STE_SIZE_MASK = 16,
87 DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK,
88 };
89
90 enum mlx5dr_ste_ctx_action_cap {
91 DR_STE_CTX_ACTION_CAP_NONE = 0,
92 DR_STE_CTX_ACTION_CAP_TX_POP = 1 << 0,
93 DR_STE_CTX_ACTION_CAP_RX_PUSH = 1 << 1,
94 DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 2,
95 DR_STE_CTX_ACTION_CAP_POP_MDFY = 1 << 3,
96 };
97
98 enum {
99 DR_MODIFY_ACTION_SIZE = 8,
100 };
101
102 enum mlx5dr_matcher_criteria {
103 DR_MATCHER_CRITERIA_EMPTY = 0,
104 DR_MATCHER_CRITERIA_OUTER = 1 << 0,
105 DR_MATCHER_CRITERIA_MISC = 1 << 1,
106 DR_MATCHER_CRITERIA_INNER = 1 << 2,
107 DR_MATCHER_CRITERIA_MISC2 = 1 << 3,
108 DR_MATCHER_CRITERIA_MISC3 = 1 << 4,
109 DR_MATCHER_CRITERIA_MISC4 = 1 << 5,
110 DR_MATCHER_CRITERIA_MISC5 = 1 << 6,
111 DR_MATCHER_CRITERIA_MAX = 1 << 7,
112 };
113
114 enum mlx5dr_action_type {
115 DR_ACTION_TYP_TNL_L2_TO_L2,
116 DR_ACTION_TYP_L2_TO_TNL_L2,
117 DR_ACTION_TYP_TNL_L3_TO_L2,
118 DR_ACTION_TYP_L2_TO_TNL_L3,
119 DR_ACTION_TYP_DROP,
120 DR_ACTION_TYP_QP,
121 DR_ACTION_TYP_FT,
122 DR_ACTION_TYP_CTR,
123 DR_ACTION_TYP_TAG,
124 DR_ACTION_TYP_MODIFY_HDR,
125 DR_ACTION_TYP_VPORT,
126 DR_ACTION_TYP_POP_VLAN,
127 DR_ACTION_TYP_PUSH_VLAN,
128 DR_ACTION_TYP_INSERT_HDR,
129 DR_ACTION_TYP_REMOVE_HDR,
130 DR_ACTION_TYP_SAMPLER,
131 DR_ACTION_TYP_ASO_FLOW_METER,
132 DR_ACTION_TYP_RANGE,
133 DR_ACTION_TYP_MAX,
134 };
135
136 enum mlx5dr_ipv {
137 DR_RULE_IPV4,
138 DR_RULE_IPV6,
139 DR_RULE_IPV_MAX,
140 };
141
142 struct mlx5dr_icm_pool;
143 struct mlx5dr_icm_chunk;
144 struct mlx5dr_icm_buddy_mem;
145 struct mlx5dr_ste_htbl;
146 struct mlx5dr_match_param;
147 struct mlx5dr_cmd_caps;
148 struct mlx5dr_rule_rx_tx;
149 struct mlx5dr_matcher_rx_tx;
150 struct mlx5dr_ste_ctx;
151 struct mlx5dr_send_info_pool;
152 struct mlx5dr_icm_hot_chunk;
153
154 struct mlx5dr_ste {
155 /* refcount: indicates the num of rules that using this ste */
156 u32 refcount;
157
158 /* this ste is part of a rule, located in ste's chain */
159 u8 ste_chain_location;
160
161 /* attached to the miss_list head at each htbl entry */
162 struct list_head miss_list_node;
163
164 /* this ste is member of htbl */
165 struct mlx5dr_ste_htbl *htbl;
166
167 struct mlx5dr_ste_htbl *next_htbl;
168
169 /* The rule this STE belongs to */
170 struct mlx5dr_rule_rx_tx *rule_rx_tx;
171 };
172
173 struct mlx5dr_ste_htbl_ctrl {
174 /* total number of valid entries belonging to this hash table. This
175 * includes the non collision and collision entries
176 */
177 unsigned int num_of_valid_entries;
178
179 /* total number of collisions entries attached to this table */
180 unsigned int num_of_collisions;
181 };
182
183 struct mlx5dr_ste_htbl {
184 u16 lu_type;
185 u16 byte_mask;
186 u32 refcount;
187 struct mlx5dr_icm_chunk *chunk;
188 struct mlx5dr_ste *pointing_ste;
189 struct mlx5dr_ste_htbl_ctrl ctrl;
190 };
191
192 struct mlx5dr_ste_send_info {
193 struct mlx5dr_ste *ste;
194 struct list_head send_list;
195 u16 size;
196 u16 offset;
197 u8 data_cont[DR_STE_SIZE];
198 u8 *data;
199 };
200
201 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size,
202 u16 offset, u8 *data,
203 struct mlx5dr_ste_send_info *ste_info,
204 struct list_head *send_list,
205 bool copy_data);
206
207 struct mlx5dr_ste_build {
208 u8 inner:1;
209 u8 rx:1;
210 u8 vhca_id_valid:1;
211 struct mlx5dr_domain *dmn;
212 struct mlx5dr_cmd_caps *caps;
213 u16 lu_type;
214 u16 byte_mask;
215 u8 bit_mask[DR_STE_SIZE_MASK];
216 int (*ste_build_tag_func)(struct mlx5dr_match_param *spec,
217 struct mlx5dr_ste_build *sb,
218 u8 *tag);
219 };
220
221 struct mlx5dr_ste_htbl *
222 mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool,
223 enum mlx5dr_icm_chunk_size chunk_size,
224 u16 lu_type, u16 byte_mask);
225
226 int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl);
227
mlx5dr_htbl_put(struct mlx5dr_ste_htbl * htbl)228 static inline void mlx5dr_htbl_put(struct mlx5dr_ste_htbl *htbl)
229 {
230 htbl->refcount--;
231 if (!htbl->refcount)
232 mlx5dr_ste_htbl_free(htbl);
233 }
234
mlx5dr_htbl_get(struct mlx5dr_ste_htbl * htbl)235 static inline void mlx5dr_htbl_get(struct mlx5dr_ste_htbl *htbl)
236 {
237 htbl->refcount++;
238 }
239
240 /* STE utils */
241 u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl);
242 bool mlx5dr_ste_is_miss_addr_set(struct mlx5dr_ste_ctx *ste_ctx, u8 *hw_ste_p);
243 void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx,
244 u8 *hw_ste, u64 miss_addr);
245 void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx,
246 u8 *hw_ste, u64 icm_addr, u32 ht_size);
247 void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx,
248 u8 *hw_ste,
249 struct mlx5dr_ste_htbl *next_htbl);
250 void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask);
251 bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher,
252 u8 ste_location);
253 u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste);
254 u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste);
255 struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste);
256
257 #define MLX5DR_MAX_VLANS 2
258
259 struct mlx5dr_ste_actions_attr {
260 u32 modify_index;
261 u16 modify_actions;
262 u32 decap_index;
263 u16 decap_actions;
264 u8 decap_with_vlan:1;
265 u64 final_icm_addr;
266 u32 flow_tag;
267 u32 ctr_id;
268 u16 gvmi;
269 u16 hit_gvmi;
270 struct {
271 u32 id;
272 u32 size;
273 u8 param_0;
274 u8 param_1;
275 } reformat;
276 struct {
277 int count;
278 u32 headers[MLX5DR_MAX_VLANS];
279 } vlans;
280
281 struct {
282 u32 obj_id;
283 u32 offset;
284 u8 dest_reg_id;
285 u8 init_color;
286 } aso_flow_meter;
287
288 struct {
289 u64 miss_icm_addr;
290 u32 definer_id;
291 u32 min;
292 u32 max;
293 } range;
294 };
295
296 void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx,
297 struct mlx5dr_domain *dmn,
298 u8 *action_type_set,
299 u8 *last_ste,
300 struct mlx5dr_ste_actions_attr *attr,
301 u32 *added_stes);
302 void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx,
303 struct mlx5dr_domain *dmn,
304 u8 *action_type_set,
305 u8 *last_ste,
306 struct mlx5dr_ste_actions_attr *attr,
307 u32 *added_stes);
308
309 void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx,
310 __be64 *hw_action,
311 u8 hw_field,
312 u8 shifter,
313 u8 length,
314 u32 data);
315 void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx,
316 __be64 *hw_action,
317 u8 hw_field,
318 u8 shifter,
319 u8 length,
320 u32 data);
321 void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx,
322 __be64 *hw_action,
323 u8 dst_hw_field,
324 u8 dst_shifter,
325 u8 dst_len,
326 u8 src_hw_field,
327 u8 src_shifter);
328 int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
329 void *data,
330 u32 data_sz,
331 u8 *hw_action,
332 u32 hw_action_sz,
333 u16 *used_hw_action_num);
334
335 const struct mlx5dr_ste_action_modify_field *
336 mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field);
337
338 struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version);
339 void mlx5dr_ste_free(struct mlx5dr_ste *ste,
340 struct mlx5dr_matcher *matcher,
341 struct mlx5dr_matcher_rx_tx *nic_matcher);
mlx5dr_ste_put(struct mlx5dr_ste * ste,struct mlx5dr_matcher * matcher,struct mlx5dr_matcher_rx_tx * nic_matcher)342 static inline void mlx5dr_ste_put(struct mlx5dr_ste *ste,
343 struct mlx5dr_matcher *matcher,
344 struct mlx5dr_matcher_rx_tx *nic_matcher)
345 {
346 ste->refcount--;
347 if (!ste->refcount)
348 mlx5dr_ste_free(ste, matcher, nic_matcher);
349 }
350
351 /* initial as 0, increased only when ste appears in a new rule */
mlx5dr_ste_get(struct mlx5dr_ste * ste)352 static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste)
353 {
354 ste->refcount++;
355 }
356
mlx5dr_ste_is_not_used(struct mlx5dr_ste * ste)357 static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste)
358 {
359 return !ste->refcount;
360 }
361
362 bool mlx5dr_ste_equal_tag(void *src, void *dst);
363 int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher,
364 struct mlx5dr_matcher_rx_tx *nic_matcher,
365 struct mlx5dr_ste *ste,
366 u8 *cur_hw_ste,
367 enum mlx5dr_icm_chunk_size log_table_size);
368
369 /* STE build functions */
370 int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn,
371 u8 match_criteria,
372 struct mlx5dr_match_param *mask,
373 struct mlx5dr_match_param *value);
374 int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher,
375 struct mlx5dr_matcher_rx_tx *nic_matcher,
376 struct mlx5dr_match_param *value,
377 u8 *ste_arr);
378 void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx,
379 struct mlx5dr_ste_build *builder,
380 struct mlx5dr_match_param *mask,
381 bool inner, bool rx);
382 void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx,
383 struct mlx5dr_ste_build *sb,
384 struct mlx5dr_match_param *mask,
385 bool inner, bool rx);
386 void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx,
387 struct mlx5dr_ste_build *sb,
388 struct mlx5dr_match_param *mask,
389 bool inner, bool rx);
390 void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx,
391 struct mlx5dr_ste_build *sb,
392 struct mlx5dr_match_param *mask,
393 bool inner, bool rx);
394 void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx,
395 struct mlx5dr_ste_build *sb,
396 struct mlx5dr_match_param *mask,
397 bool inner, bool rx);
398 void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx,
399 struct mlx5dr_ste_build *sb,
400 struct mlx5dr_match_param *mask,
401 bool inner, bool rx);
402 void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx,
403 struct mlx5dr_ste_build *sb,
404 struct mlx5dr_match_param *mask,
405 bool inner, bool rx);
406 void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx,
407 struct mlx5dr_ste_build *sb,
408 struct mlx5dr_match_param *mask,
409 bool inner, bool rx);
410 void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx,
411 struct mlx5dr_ste_build *sb,
412 struct mlx5dr_match_param *mask,
413 bool inner, bool rx);
414 void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx,
415 struct mlx5dr_ste_build *sb,
416 struct mlx5dr_match_param *mask,
417 bool inner, bool rx);
418 void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx,
419 struct mlx5dr_ste_build *sb,
420 struct mlx5dr_match_param *mask,
421 bool inner, bool rx);
422 void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx,
423 struct mlx5dr_ste_build *sb,
424 struct mlx5dr_match_param *mask,
425 bool inner, bool rx);
426 void mlx5dr_ste_build_tnl_mpls(struct mlx5dr_ste_ctx *ste_ctx,
427 struct mlx5dr_ste_build *sb,
428 struct mlx5dr_match_param *mask,
429 bool inner, bool rx);
430 void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx,
431 struct mlx5dr_ste_build *sb,
432 struct mlx5dr_match_param *mask,
433 struct mlx5dr_cmd_caps *caps,
434 bool inner, bool rx);
435 void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx,
436 struct mlx5dr_ste_build *sb,
437 struct mlx5dr_match_param *mask,
438 struct mlx5dr_cmd_caps *caps,
439 bool inner, bool rx);
440 void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx,
441 struct mlx5dr_ste_build *sb,
442 struct mlx5dr_match_param *mask,
443 struct mlx5dr_cmd_caps *caps,
444 bool inner, bool rx);
445 void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx,
446 struct mlx5dr_ste_build *sb,
447 struct mlx5dr_match_param *mask,
448 bool inner, bool rx);
449 void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx,
450 struct mlx5dr_ste_build *sb,
451 struct mlx5dr_match_param *mask,
452 bool inner, bool rx);
453 void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx,
454 struct mlx5dr_ste_build *sb,
455 struct mlx5dr_match_param *mask,
456 struct mlx5dr_cmd_caps *caps,
457 bool inner, bool rx);
458 void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx,
459 struct mlx5dr_ste_build *sb,
460 struct mlx5dr_match_param *mask,
461 struct mlx5dr_cmd_caps *caps,
462 bool inner, bool rx);
463 void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx,
464 struct mlx5dr_ste_build *sb,
465 struct mlx5dr_match_param *mask,
466 bool inner, bool rx);
467 void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
468 struct mlx5dr_ste_build *sb,
469 struct mlx5dr_match_param *mask,
470 struct mlx5dr_cmd_caps *caps,
471 bool inner, bool rx);
472 void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
473 struct mlx5dr_ste_build *sb,
474 struct mlx5dr_match_param *mask,
475 struct mlx5dr_cmd_caps *caps,
476 bool inner, bool rx);
477 void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx,
478 struct mlx5dr_ste_build *sb,
479 struct mlx5dr_match_param *mask,
480 bool inner, bool rx);
481 void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx,
482 struct mlx5dr_ste_build *sb,
483 struct mlx5dr_match_param *mask,
484 bool inner, bool rx);
485 void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx,
486 struct mlx5dr_ste_build *sb,
487 struct mlx5dr_match_param *mask,
488 bool inner, bool rx);
489 void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx,
490 struct mlx5dr_ste_build *sb,
491 struct mlx5dr_match_param *mask,
492 bool inner, bool rx);
493 void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx,
494 struct mlx5dr_ste_build *sb,
495 struct mlx5dr_match_param *mask,
496 struct mlx5dr_domain *dmn,
497 bool inner, bool rx);
498 void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
499 struct mlx5dr_ste_build *sb,
500 struct mlx5dr_match_param *mask,
501 bool inner, bool rx);
502 void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
503 struct mlx5dr_ste_build *sb,
504 struct mlx5dr_match_param *mask,
505 bool inner, bool rx);
506 void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx);
507
508 /* Actions utils */
509 int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher,
510 struct mlx5dr_matcher_rx_tx *nic_matcher,
511 struct mlx5dr_action *actions[],
512 u32 num_actions,
513 u8 *ste_arr,
514 u32 *new_hw_ste_arr_sz);
515
516 struct mlx5dr_match_spec {
517 u32 smac_47_16; /* Source MAC address of incoming packet */
518 /* Incoming packet Ethertype - this is the Ethertype
519 * following the last VLAN tag of the packet
520 */
521 u32 smac_15_0:16; /* Source MAC address of incoming packet */
522 u32 ethertype:16;
523
524 u32 dmac_47_16; /* Destination MAC address of incoming packet */
525
526 u32 dmac_15_0:16; /* Destination MAC address of incoming packet */
527 /* Priority of first VLAN tag in the incoming packet.
528 * Valid only when cvlan_tag==1 or svlan_tag==1
529 */
530 u32 first_prio:3;
531 /* CFI bit of first VLAN tag in the incoming packet.
532 * Valid only when cvlan_tag==1 or svlan_tag==1
533 */
534 u32 first_cfi:1;
535 /* VLAN ID of first VLAN tag in the incoming packet.
536 * Valid only when cvlan_tag==1 or svlan_tag==1
537 */
538 u32 first_vid:12;
539
540 u32 ip_protocol:8; /* IP protocol */
541 /* Differentiated Services Code Point derived from
542 * Traffic Class/TOS field of IPv6/v4
543 */
544 u32 ip_dscp:6;
545 /* Explicit Congestion Notification derived from
546 * Traffic Class/TOS field of IPv6/v4
547 */
548 u32 ip_ecn:2;
549 /* The first vlan in the packet is c-vlan (0x8100).
550 * cvlan_tag and svlan_tag cannot be set together
551 */
552 u32 cvlan_tag:1;
553 /* The first vlan in the packet is s-vlan (0x8a88).
554 * cvlan_tag and svlan_tag cannot be set together
555 */
556 u32 svlan_tag:1;
557 u32 frag:1; /* Packet is an IP fragment */
558 u32 ip_version:4; /* IP version */
559 /* TCP flags. ;Bit 0: FIN;Bit 1: SYN;Bit 2: RST;Bit 3: PSH;Bit 4: ACK;
560 * Bit 5: URG;Bit 6: ECE;Bit 7: CWR;Bit 8: NS
561 */
562 u32 tcp_flags:9;
563
564 /* TCP source port.;tcp and udp sport/dport are mutually exclusive */
565 u32 tcp_sport:16;
566 /* TCP destination port.
567 * tcp and udp sport/dport are mutually exclusive
568 */
569 u32 tcp_dport:16;
570
571 u32 reserved_auto1:16;
572 u32 ipv4_ihl:4;
573 u32 reserved_auto2:4;
574 u32 ttl_hoplimit:8;
575
576 /* UDP source port.;tcp and udp sport/dport are mutually exclusive */
577 u32 udp_sport:16;
578 /* UDP destination port.;tcp and udp sport/dport are mutually exclusive */
579 u32 udp_dport:16;
580
581 /* IPv6 source address of incoming packets
582 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
583 * This field should be qualified by an appropriate ethertype
584 */
585 u32 src_ip_127_96;
586 /* IPv6 source address of incoming packets
587 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
588 * This field should be qualified by an appropriate ethertype
589 */
590 u32 src_ip_95_64;
591 /* IPv6 source address of incoming packets
592 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
593 * This field should be qualified by an appropriate ethertype
594 */
595 u32 src_ip_63_32;
596 /* IPv6 source address of incoming packets
597 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
598 * This field should be qualified by an appropriate ethertype
599 */
600 u32 src_ip_31_0;
601 /* IPv6 destination address of incoming packets
602 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
603 * This field should be qualified by an appropriate ethertype
604 */
605 u32 dst_ip_127_96;
606 /* IPv6 destination address of incoming packets
607 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
608 * This field should be qualified by an appropriate ethertype
609 */
610 u32 dst_ip_95_64;
611 /* IPv6 destination address of incoming packets
612 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
613 * This field should be qualified by an appropriate ethertype
614 */
615 u32 dst_ip_63_32;
616 /* IPv6 destination address of incoming packets
617 * For IPv4 address use bits 31:0 (rest of the bits are reserved)
618 * This field should be qualified by an appropriate ethertype
619 */
620 u32 dst_ip_31_0;
621 };
622
623 struct mlx5dr_match_misc {
624 /* used with GRE, checksum exist when gre_c_present == 1 */
625 u32 gre_c_present:1;
626 u32 reserved_auto1:1;
627 /* used with GRE, key exist when gre_k_present == 1 */
628 u32 gre_k_present:1;
629 /* used with GRE, sequence number exist when gre_s_present == 1 */
630 u32 gre_s_present:1;
631 u32 source_vhca_port:4;
632 u32 source_sqn:24; /* Source SQN */
633
634 u32 source_eswitch_owner_vhca_id:16;
635 /* Source port.;0xffff determines wire port */
636 u32 source_port:16;
637
638 /* Priority of second VLAN tag in the outer header of the incoming packet.
639 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
640 */
641 u32 outer_second_prio:3;
642 /* CFI bit of first VLAN tag in the outer header of the incoming packet.
643 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
644 */
645 u32 outer_second_cfi:1;
646 /* VLAN ID of first VLAN tag the outer header of the incoming packet.
647 * Valid only when outer_second_cvlan_tag ==1 or outer_second_svlan_tag ==1
648 */
649 u32 outer_second_vid:12;
650 /* Priority of second VLAN tag in the inner header of the incoming packet.
651 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
652 */
653 u32 inner_second_prio:3;
654 /* CFI bit of first VLAN tag in the inner header of the incoming packet.
655 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
656 */
657 u32 inner_second_cfi:1;
658 /* VLAN ID of first VLAN tag the inner header of the incoming packet.
659 * Valid only when inner_second_cvlan_tag ==1 or inner_second_svlan_tag ==1
660 */
661 u32 inner_second_vid:12;
662
663 u32 outer_second_cvlan_tag:1;
664 u32 inner_second_cvlan_tag:1;
665 /* The second vlan in the outer header of the packet is c-vlan (0x8100).
666 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
667 */
668 u32 outer_second_svlan_tag:1;
669 /* The second vlan in the inner header of the packet is c-vlan (0x8100).
670 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
671 */
672 u32 inner_second_svlan_tag:1;
673 /* The second vlan in the outer header of the packet is s-vlan (0x8a88).
674 * outer_second_cvlan_tag and outer_second_svlan_tag cannot be set together
675 */
676 u32 reserved_auto2:12;
677 /* The second vlan in the inner header of the packet is s-vlan (0x8a88).
678 * inner_second_cvlan_tag and inner_second_svlan_tag cannot be set together
679 */
680 u32 gre_protocol:16; /* GRE Protocol (outer) */
681
682 u32 gre_key_h:24; /* GRE Key[31:8] (outer) */
683 u32 gre_key_l:8; /* GRE Key [7:0] (outer) */
684
685 u32 vxlan_vni:24; /* VXLAN VNI (outer) */
686 u32 reserved_auto3:8;
687
688 u32 geneve_vni:24; /* GENEVE VNI field (outer) */
689 u32 reserved_auto4:6;
690 u32 geneve_tlv_option_0_exist:1;
691 u32 geneve_oam:1; /* GENEVE OAM field (outer) */
692
693 u32 reserved_auto5:12;
694 u32 outer_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (outer) */
695
696 u32 reserved_auto6:12;
697 u32 inner_ipv6_flow_label:20; /* Flow label of incoming IPv6 packet (inner) */
698
699 u32 reserved_auto7:10;
700 u32 geneve_opt_len:6; /* GENEVE OptLen (outer) */
701 u32 geneve_protocol_type:16; /* GENEVE protocol type (outer) */
702
703 u32 reserved_auto8:8;
704 u32 bth_dst_qp:24; /* Destination QP in BTH header */
705
706 u32 reserved_auto9;
707 u32 outer_esp_spi;
708 u32 reserved_auto10[3];
709 };
710
711 struct mlx5dr_match_misc2 {
712 u32 outer_first_mpls_label:20; /* First MPLS LABEL (outer) */
713 u32 outer_first_mpls_exp:3; /* First MPLS EXP (outer) */
714 u32 outer_first_mpls_s_bos:1; /* First MPLS S_BOS (outer) */
715 u32 outer_first_mpls_ttl:8; /* First MPLS TTL (outer) */
716
717 u32 inner_first_mpls_label:20; /* First MPLS LABEL (inner) */
718 u32 inner_first_mpls_exp:3; /* First MPLS EXP (inner) */
719 u32 inner_first_mpls_s_bos:1; /* First MPLS S_BOS (inner) */
720 u32 inner_first_mpls_ttl:8; /* First MPLS TTL (inner) */
721
722 u32 outer_first_mpls_over_gre_label:20; /* last MPLS LABEL (outer) */
723 u32 outer_first_mpls_over_gre_exp:3; /* last MPLS EXP (outer) */
724 u32 outer_first_mpls_over_gre_s_bos:1; /* last MPLS S_BOS (outer) */
725 u32 outer_first_mpls_over_gre_ttl:8; /* last MPLS TTL (outer) */
726
727 u32 outer_first_mpls_over_udp_label:20; /* last MPLS LABEL (outer) */
728 u32 outer_first_mpls_over_udp_exp:3; /* last MPLS EXP (outer) */
729 u32 outer_first_mpls_over_udp_s_bos:1; /* last MPLS S_BOS (outer) */
730 u32 outer_first_mpls_over_udp_ttl:8; /* last MPLS TTL (outer) */
731
732 u32 metadata_reg_c_7; /* metadata_reg_c_7 */
733 u32 metadata_reg_c_6; /* metadata_reg_c_6 */
734 u32 metadata_reg_c_5; /* metadata_reg_c_5 */
735 u32 metadata_reg_c_4; /* metadata_reg_c_4 */
736 u32 metadata_reg_c_3; /* metadata_reg_c_3 */
737 u32 metadata_reg_c_2; /* metadata_reg_c_2 */
738 u32 metadata_reg_c_1; /* metadata_reg_c_1 */
739 u32 metadata_reg_c_0; /* metadata_reg_c_0 */
740 u32 metadata_reg_a; /* metadata_reg_a */
741 u32 reserved_auto1[3];
742 };
743
744 struct mlx5dr_match_misc3 {
745 u32 inner_tcp_seq_num;
746 u32 outer_tcp_seq_num;
747 u32 inner_tcp_ack_num;
748 u32 outer_tcp_ack_num;
749
750 u32 reserved_auto1:8;
751 u32 outer_vxlan_gpe_vni:24;
752
753 u32 outer_vxlan_gpe_next_protocol:8;
754 u32 outer_vxlan_gpe_flags:8;
755 u32 reserved_auto2:16;
756
757 u32 icmpv4_header_data;
758 u32 icmpv6_header_data;
759
760 u8 icmpv4_type;
761 u8 icmpv4_code;
762 u8 icmpv6_type;
763 u8 icmpv6_code;
764
765 u32 geneve_tlv_option_0_data;
766
767 u32 gtpu_teid;
768
769 u8 gtpu_msg_type;
770 u8 gtpu_msg_flags;
771 u32 reserved_auto3:16;
772
773 u32 gtpu_dw_2;
774 u32 gtpu_first_ext_dw_0;
775 u32 gtpu_dw_0;
776 u32 reserved_auto4;
777 };
778
779 struct mlx5dr_match_misc4 {
780 u32 prog_sample_field_value_0;
781 u32 prog_sample_field_id_0;
782 u32 prog_sample_field_value_1;
783 u32 prog_sample_field_id_1;
784 u32 prog_sample_field_value_2;
785 u32 prog_sample_field_id_2;
786 u32 prog_sample_field_value_3;
787 u32 prog_sample_field_id_3;
788 u32 reserved_auto1[8];
789 };
790
791 struct mlx5dr_match_misc5 {
792 u32 macsec_tag_0;
793 u32 macsec_tag_1;
794 u32 macsec_tag_2;
795 u32 macsec_tag_3;
796 u32 tunnel_header_0;
797 u32 tunnel_header_1;
798 u32 tunnel_header_2;
799 u32 tunnel_header_3;
800 };
801
802 struct mlx5dr_match_param {
803 struct mlx5dr_match_spec outer;
804 struct mlx5dr_match_misc misc;
805 struct mlx5dr_match_spec inner;
806 struct mlx5dr_match_misc2 misc2;
807 struct mlx5dr_match_misc3 misc3;
808 struct mlx5dr_match_misc4 misc4;
809 struct mlx5dr_match_misc5 misc5;
810 };
811
812 #define DR_MASK_IS_ICMPV4_SET(_misc3) ((_misc3)->icmpv4_type || \
813 (_misc3)->icmpv4_code || \
814 (_misc3)->icmpv4_header_data)
815
816 #define DR_MASK_IS_SRC_IP_SET(_spec) ((_spec)->src_ip_127_96 || \
817 (_spec)->src_ip_95_64 || \
818 (_spec)->src_ip_63_32 || \
819 (_spec)->src_ip_31_0)
820
821 #define DR_MASK_IS_DST_IP_SET(_spec) ((_spec)->dst_ip_127_96 || \
822 (_spec)->dst_ip_95_64 || \
823 (_spec)->dst_ip_63_32 || \
824 (_spec)->dst_ip_31_0)
825
826 struct mlx5dr_esw_caps {
827 u64 drop_icm_address_rx;
828 u64 drop_icm_address_tx;
829 u64 uplink_icm_address_rx;
830 u64 uplink_icm_address_tx;
831 u8 sw_owner:1;
832 u8 sw_owner_v2:1;
833 };
834
835 struct mlx5dr_cmd_vport_cap {
836 u16 vport_gvmi;
837 u16 vhca_gvmi;
838 u16 num;
839 u64 icm_address_rx;
840 u64 icm_address_tx;
841 };
842
843 struct mlx5dr_roce_cap {
844 u8 roce_en:1;
845 u8 fl_rc_qp_when_roce_disabled:1;
846 u8 fl_rc_qp_when_roce_enabled:1;
847 };
848
849 struct mlx5dr_vports {
850 struct mlx5dr_cmd_vport_cap esw_manager_caps;
851 struct mlx5dr_cmd_vport_cap uplink_caps;
852 struct xarray vports_caps_xa;
853 };
854
855 struct mlx5dr_cmd_caps {
856 u16 gvmi;
857 u64 nic_rx_drop_address;
858 u64 nic_tx_drop_address;
859 u64 nic_tx_allow_address;
860 u64 esw_rx_drop_address;
861 u64 esw_tx_drop_address;
862 u32 log_icm_size;
863 u64 hdr_modify_icm_addr;
864 u32 flex_protocols;
865 u8 flex_parser_id_icmp_dw0;
866 u8 flex_parser_id_icmp_dw1;
867 u8 flex_parser_id_icmpv6_dw0;
868 u8 flex_parser_id_icmpv6_dw1;
869 u8 flex_parser_id_geneve_tlv_option_0;
870 u8 flex_parser_id_mpls_over_gre;
871 u8 flex_parser_id_mpls_over_udp;
872 u8 flex_parser_id_gtpu_dw_0;
873 u8 flex_parser_id_gtpu_teid;
874 u8 flex_parser_id_gtpu_dw_2;
875 u8 flex_parser_id_gtpu_first_ext_dw_0;
876 u8 flex_parser_ok_bits_supp;
877 u8 max_ft_level;
878 u16 roce_min_src_udp;
879 u8 sw_format_ver;
880 bool eswitch_manager;
881 bool rx_sw_owner;
882 bool tx_sw_owner;
883 bool fdb_sw_owner;
884 u8 rx_sw_owner_v2:1;
885 u8 tx_sw_owner_v2:1;
886 u8 fdb_sw_owner_v2:1;
887 struct mlx5dr_esw_caps esw_caps;
888 struct mlx5dr_vports vports;
889 bool prio_tag_required;
890 struct mlx5dr_roce_cap roce_caps;
891 u8 is_ecpf:1;
892 u8 isolate_vl_tc:1;
893 };
894
895 enum mlx5dr_domain_nic_type {
896 DR_DOMAIN_NIC_TYPE_RX,
897 DR_DOMAIN_NIC_TYPE_TX,
898 };
899
900 struct mlx5dr_domain_rx_tx {
901 u64 drop_icm_addr;
902 u64 default_icm_addr;
903 enum mlx5dr_domain_nic_type type;
904 struct mutex mutex; /* protect rx/tx domain */
905 };
906
907 struct mlx5dr_domain_info {
908 bool supp_sw_steering;
909 u32 max_inline_size;
910 u32 max_send_wr;
911 u32 max_log_sw_icm_sz;
912 u32 max_log_action_icm_sz;
913 struct mlx5dr_domain_rx_tx rx;
914 struct mlx5dr_domain_rx_tx tx;
915 struct mlx5dr_cmd_caps caps;
916 };
917
918 struct mlx5dr_domain {
919 struct mlx5dr_domain *peer_dmn;
920 struct mlx5_core_dev *mdev;
921 u32 pdn;
922 struct mlx5_uars_page *uar;
923 enum mlx5dr_domain_type type;
924 refcount_t refcount;
925 struct mlx5dr_icm_pool *ste_icm_pool;
926 struct mlx5dr_icm_pool *action_icm_pool;
927 struct mlx5dr_send_info_pool *send_info_pool_rx;
928 struct mlx5dr_send_info_pool *send_info_pool_tx;
929 struct kmem_cache *chunks_kmem_cache;
930 struct kmem_cache *htbls_kmem_cache;
931 struct mlx5dr_send_ring *send_ring;
932 struct mlx5dr_domain_info info;
933 struct xarray csum_fts_xa;
934 struct mlx5dr_ste_ctx *ste_ctx;
935 struct list_head dbg_tbl_list;
936 struct mlx5dr_dbg_dump_info dump_info;
937 struct xarray definers_xa;
938 };
939
940 struct mlx5dr_table_rx_tx {
941 struct mlx5dr_ste_htbl *s_anchor;
942 struct mlx5dr_domain_rx_tx *nic_dmn;
943 u64 default_icm_addr;
944 struct list_head nic_matcher_list;
945 };
946
947 struct mlx5dr_table {
948 struct mlx5dr_domain *dmn;
949 struct mlx5dr_table_rx_tx rx;
950 struct mlx5dr_table_rx_tx tx;
951 u32 level;
952 u32 table_type;
953 u32 table_id;
954 u32 flags;
955 struct list_head matcher_list;
956 struct mlx5dr_action *miss_action;
957 refcount_t refcount;
958 struct list_head dbg_node;
959 };
960
961 struct mlx5dr_matcher_rx_tx {
962 struct mlx5dr_ste_htbl *s_htbl;
963 struct mlx5dr_ste_htbl *e_anchor;
964 struct mlx5dr_ste_build *ste_builder;
965 struct mlx5dr_ste_build ste_builder_arr[DR_RULE_IPV_MAX]
966 [DR_RULE_IPV_MAX]
967 [DR_RULE_MAX_STES];
968 u8 num_of_builders;
969 u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX];
970 u64 default_icm_addr;
971 struct mlx5dr_table_rx_tx *nic_tbl;
972 u32 prio;
973 struct list_head list_node;
974 u32 rules;
975 };
976
977 struct mlx5dr_matcher {
978 struct mlx5dr_table *tbl;
979 struct mlx5dr_matcher_rx_tx rx;
980 struct mlx5dr_matcher_rx_tx tx;
981 struct list_head list_node; /* Used for both matchers and dbg managing */
982 u32 prio;
983 struct mlx5dr_match_param mask;
984 u8 match_criteria;
985 refcount_t refcount;
986 struct list_head dbg_rule_list;
987 };
988
989 struct mlx5dr_ste_action_modify_field {
990 u16 hw_field;
991 u8 start;
992 u8 end;
993 u8 l3_type;
994 u8 l4_type;
995 };
996
997 struct mlx5dr_action_rewrite {
998 struct mlx5dr_domain *dmn;
999 struct mlx5dr_icm_chunk *chunk;
1000 u8 *data;
1001 u16 num_of_actions;
1002 u32 index;
1003 u8 allow_rx:1;
1004 u8 allow_tx:1;
1005 u8 modify_ttl:1;
1006 };
1007
1008 struct mlx5dr_action_reformat {
1009 struct mlx5dr_domain *dmn;
1010 u32 id;
1011 u32 size;
1012 u8 param_0;
1013 u8 param_1;
1014 };
1015
1016 struct mlx5dr_action_sampler {
1017 struct mlx5dr_domain *dmn;
1018 u64 rx_icm_addr;
1019 u64 tx_icm_addr;
1020 u32 sampler_id;
1021 };
1022
1023 struct mlx5dr_action_dest_tbl {
1024 u8 is_fw_tbl:1;
1025 union {
1026 struct mlx5dr_table *tbl;
1027 struct {
1028 struct mlx5dr_domain *dmn;
1029 u32 id;
1030 u32 group_id;
1031 enum fs_flow_table_type type;
1032 u64 rx_icm_addr;
1033 u64 tx_icm_addr;
1034 struct mlx5dr_action **ref_actions;
1035 u32 num_of_ref_actions;
1036 } fw_tbl;
1037 };
1038 };
1039
1040 struct mlx5dr_action_range {
1041 struct mlx5dr_domain *dmn;
1042 struct mlx5dr_action *hit_tbl_action;
1043 struct mlx5dr_action *miss_tbl_action;
1044 u32 definer_id;
1045 u32 min;
1046 u32 max;
1047 };
1048
1049 struct mlx5dr_action_ctr {
1050 u32 ctr_id;
1051 u32 offset;
1052 };
1053
1054 struct mlx5dr_action_vport {
1055 struct mlx5dr_domain *dmn;
1056 struct mlx5dr_cmd_vport_cap *caps;
1057 };
1058
1059 struct mlx5dr_action_push_vlan {
1060 u32 vlan_hdr; /* tpid_pcp_dei_vid */
1061 };
1062
1063 struct mlx5dr_action_flow_tag {
1064 u32 flow_tag;
1065 };
1066
1067 struct mlx5dr_rule_action_member {
1068 struct mlx5dr_action *action;
1069 struct list_head list;
1070 };
1071
1072 struct mlx5dr_action_aso_flow_meter {
1073 struct mlx5dr_domain *dmn;
1074 u32 obj_id;
1075 u32 offset;
1076 u8 dest_reg_id;
1077 u8 init_color;
1078 };
1079
1080 struct mlx5dr_action {
1081 enum mlx5dr_action_type action_type;
1082 refcount_t refcount;
1083
1084 union {
1085 void *data;
1086 struct mlx5dr_action_rewrite *rewrite;
1087 struct mlx5dr_action_reformat *reformat;
1088 struct mlx5dr_action_sampler *sampler;
1089 struct mlx5dr_action_dest_tbl *dest_tbl;
1090 struct mlx5dr_action_ctr *ctr;
1091 struct mlx5dr_action_vport *vport;
1092 struct mlx5dr_action_push_vlan *push_vlan;
1093 struct mlx5dr_action_flow_tag *flow_tag;
1094 struct mlx5dr_action_aso_flow_meter *aso;
1095 struct mlx5dr_action_range *range;
1096 };
1097 };
1098
1099 enum mlx5dr_connect_type {
1100 CONNECT_HIT = 1,
1101 CONNECT_MISS = 2,
1102 };
1103
1104 struct mlx5dr_htbl_connect_info {
1105 enum mlx5dr_connect_type type;
1106 union {
1107 struct mlx5dr_ste_htbl *hit_next_htbl;
1108 u64 miss_icm_addr;
1109 };
1110 };
1111
1112 struct mlx5dr_rule_rx_tx {
1113 struct mlx5dr_matcher_rx_tx *nic_matcher;
1114 struct mlx5dr_ste *last_rule_ste;
1115 };
1116
1117 struct mlx5dr_rule {
1118 struct mlx5dr_matcher *matcher;
1119 struct mlx5dr_rule_rx_tx rx;
1120 struct mlx5dr_rule_rx_tx tx;
1121 struct list_head rule_actions_list;
1122 struct list_head dbg_node;
1123 u32 flow_source;
1124 };
1125
1126 void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule,
1127 struct mlx5dr_ste *ste,
1128 bool force);
1129 int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr,
1130 struct mlx5dr_ste *curr_ste,
1131 int *num_of_stes);
1132
1133 struct mlx5dr_icm_chunk {
1134 struct mlx5dr_icm_buddy_mem *buddy_mem;
1135
1136 /* indicates the index of this chunk in the whole memory,
1137 * used for deleting the chunk from the buddy
1138 */
1139 unsigned int seg;
1140 enum mlx5dr_icm_chunk_size size;
1141
1142 /* Memory optimisation */
1143 struct mlx5dr_ste *ste_arr;
1144 u8 *hw_ste_arr;
1145 struct list_head *miss_list;
1146 };
1147
mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx * nic_dmn)1148 static inline void mlx5dr_domain_nic_lock(struct mlx5dr_domain_rx_tx *nic_dmn)
1149 {
1150 mutex_lock(&nic_dmn->mutex);
1151 }
1152
mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx * nic_dmn)1153 static inline void mlx5dr_domain_nic_unlock(struct mlx5dr_domain_rx_tx *nic_dmn)
1154 {
1155 mutex_unlock(&nic_dmn->mutex);
1156 }
1157
mlx5dr_domain_lock(struct mlx5dr_domain * dmn)1158 static inline void mlx5dr_domain_lock(struct mlx5dr_domain *dmn)
1159 {
1160 mlx5dr_domain_nic_lock(&dmn->info.rx);
1161 mlx5dr_domain_nic_lock(&dmn->info.tx);
1162 }
1163
mlx5dr_domain_unlock(struct mlx5dr_domain * dmn)1164 static inline void mlx5dr_domain_unlock(struct mlx5dr_domain *dmn)
1165 {
1166 mlx5dr_domain_nic_unlock(&dmn->info.tx);
1167 mlx5dr_domain_nic_unlock(&dmn->info.rx);
1168 }
1169
1170 int mlx5dr_matcher_add_to_tbl_nic(struct mlx5dr_domain *dmn,
1171 struct mlx5dr_matcher_rx_tx *nic_matcher);
1172 int mlx5dr_matcher_remove_from_tbl_nic(struct mlx5dr_domain *dmn,
1173 struct mlx5dr_matcher_rx_tx *nic_matcher);
1174
1175 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher,
1176 struct mlx5dr_matcher_rx_tx *nic_matcher,
1177 enum mlx5dr_ipv outer_ipv,
1178 enum mlx5dr_ipv inner_ipv);
1179
1180 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk);
1181 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk);
1182 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk);
1183 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk);
1184 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk);
1185 u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste);
1186
1187 struct mlx5dr_ste_htbl *mlx5dr_icm_pool_alloc_htbl(struct mlx5dr_icm_pool *pool);
1188 void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_htbl *htbl);
1189
1190 static inline int
mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)1191 mlx5dr_icm_pool_dm_type_to_entry_size(enum mlx5dr_icm_type icm_type)
1192 {
1193 if (icm_type == DR_ICM_TYPE_STE)
1194 return DR_STE_SIZE;
1195
1196 return DR_MODIFY_ACTION_SIZE;
1197 }
1198
1199 static inline u32
mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)1200 mlx5dr_icm_pool_chunk_size_to_entries(enum mlx5dr_icm_chunk_size chunk_size)
1201 {
1202 return 1 << chunk_size;
1203 }
1204
1205 static inline int
mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,enum mlx5dr_icm_type icm_type)1206 mlx5dr_icm_pool_chunk_size_to_byte(enum mlx5dr_icm_chunk_size chunk_size,
1207 enum mlx5dr_icm_type icm_type)
1208 {
1209 int num_of_entries;
1210 int entry_size;
1211
1212 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(icm_type);
1213 num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
1214
1215 return entry_size * num_of_entries;
1216 }
1217
1218 static inline int
mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl * htbl)1219 mlx5dr_ste_htbl_increase_threshold(struct mlx5dr_ste_htbl *htbl)
1220 {
1221 int num_of_entries =
1222 mlx5dr_icm_pool_chunk_size_to_entries(htbl->chunk->size);
1223
1224 /* Threshold is 50%, one is added to table of size 1 */
1225 return (num_of_entries + 1) / 2;
1226 }
1227
1228 static inline bool
mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl * htbl)1229 mlx5dr_ste_htbl_may_grow(struct mlx5dr_ste_htbl *htbl)
1230 {
1231 if (htbl->chunk->size == DR_CHUNK_SIZE_MAX - 1 || !htbl->byte_mask)
1232 return false;
1233
1234 return true;
1235 }
1236
1237 struct mlx5dr_cmd_vport_cap *
1238 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport);
1239
1240 struct mlx5dr_cmd_query_flow_table_details {
1241 u8 status;
1242 u8 level;
1243 u64 sw_owner_icm_root_1;
1244 u64 sw_owner_icm_root_0;
1245 };
1246
1247 struct mlx5dr_cmd_create_flow_table_attr {
1248 u32 table_type;
1249 u16 uid;
1250 u64 icm_addr_rx;
1251 u64 icm_addr_tx;
1252 u8 level;
1253 bool sw_owner;
1254 bool term_tbl;
1255 bool decap_en;
1256 bool reformat_en;
1257 };
1258
1259 /* internal API functions */
1260 int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev,
1261 struct mlx5dr_cmd_caps *caps);
1262 int mlx5dr_cmd_query_esw_vport_context(struct mlx5_core_dev *mdev,
1263 bool other_vport, u16 vport_number,
1264 u64 *icm_address_rx,
1265 u64 *icm_address_tx);
1266 int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev,
1267 bool other_vport, u16 vport_number, u16 *gvmi);
1268 int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev,
1269 struct mlx5dr_esw_caps *caps);
1270 int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev,
1271 u32 sampler_id,
1272 u64 *rx_icm_addr,
1273 u64 *tx_icm_addr);
1274 int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev);
1275 int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev,
1276 u32 table_type,
1277 u32 table_id,
1278 u32 group_id,
1279 u32 modify_header_id,
1280 u16 vport_id);
1281 int mlx5dr_cmd_del_flow_table_entry(struct mlx5_core_dev *mdev,
1282 u32 table_type,
1283 u32 table_id);
1284 int mlx5dr_cmd_alloc_modify_header(struct mlx5_core_dev *mdev,
1285 u32 table_type,
1286 u8 num_of_actions,
1287 u64 *actions,
1288 u32 *modify_header_id);
1289 int mlx5dr_cmd_dealloc_modify_header(struct mlx5_core_dev *mdev,
1290 u32 modify_header_id);
1291 int mlx5dr_cmd_create_empty_flow_group(struct mlx5_core_dev *mdev,
1292 u32 table_type,
1293 u32 table_id,
1294 u32 *group_id);
1295 int mlx5dr_cmd_destroy_flow_group(struct mlx5_core_dev *mdev,
1296 u32 table_type,
1297 u32 table_id,
1298 u32 group_id);
1299 int mlx5dr_cmd_create_flow_table(struct mlx5_core_dev *mdev,
1300 struct mlx5dr_cmd_create_flow_table_attr *attr,
1301 u64 *fdb_rx_icm_addr,
1302 u32 *table_id);
1303 int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev,
1304 u32 table_id,
1305 u32 table_type);
1306 int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev,
1307 enum fs_flow_table_type type,
1308 u32 table_id,
1309 struct mlx5dr_cmd_query_flow_table_details *output);
1310 int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev,
1311 enum mlx5_reformat_ctx_type rt,
1312 u8 reformat_param_0,
1313 u8 reformat_param_1,
1314 size_t reformat_size,
1315 void *reformat_data,
1316 u32 *reformat_id);
1317 void mlx5dr_cmd_destroy_reformat_ctx(struct mlx5_core_dev *mdev,
1318 u32 reformat_id);
1319 int mlx5dr_cmd_create_definer(struct mlx5_core_dev *mdev,
1320 u16 format_id,
1321 u8 *dw_selectors,
1322 u8 *byte_selectors,
1323 u8 *match_mask,
1324 u32 *definer_id);
1325 void mlx5dr_cmd_destroy_definer(struct mlx5_core_dev *mdev,
1326 u32 definer_id);
1327
1328 struct mlx5dr_cmd_gid_attr {
1329 u8 gid[16];
1330 u8 mac[6];
1331 u32 roce_ver;
1332 };
1333
1334 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
1335 u16 index, struct mlx5dr_cmd_gid_attr *attr);
1336
1337 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
1338 enum mlx5dr_icm_type icm_type);
1339 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool);
1340
1341 struct mlx5dr_icm_chunk *
1342 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
1343 enum mlx5dr_icm_chunk_size chunk_size);
1344 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk);
1345
1346 void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx,
1347 u8 *hw_ste_p, u32 ste_size);
1348 int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn,
1349 struct mlx5dr_domain_rx_tx *nic_dmn,
1350 struct mlx5dr_ste_htbl *htbl,
1351 struct mlx5dr_htbl_connect_info *connect_info,
1352 bool update_hw_ste);
1353 void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx,
1354 u16 gvmi,
1355 enum mlx5dr_domain_nic_type nic_type,
1356 struct mlx5dr_ste_htbl *htbl,
1357 u8 *formatted_ste,
1358 struct mlx5dr_htbl_connect_info *connect_info);
1359 void mlx5dr_ste_copy_param(u8 match_criteria,
1360 struct mlx5dr_match_param *set_param,
1361 struct mlx5dr_match_parameters *mask,
1362 bool clear);
1363
1364 struct mlx5dr_qp {
1365 struct mlx5_core_dev *mdev;
1366 struct mlx5_wq_qp wq;
1367 struct mlx5_uars_page *uar;
1368 struct mlx5_wq_ctrl wq_ctrl;
1369 u32 qpn;
1370 struct {
1371 unsigned int pc;
1372 unsigned int cc;
1373 unsigned int size;
1374 unsigned int *wqe_head;
1375 unsigned int wqe_cnt;
1376 } sq;
1377 struct {
1378 unsigned int pc;
1379 unsigned int cc;
1380 unsigned int size;
1381 unsigned int wqe_cnt;
1382 } rq;
1383 int max_inline_data;
1384 };
1385
1386 struct mlx5dr_cq {
1387 struct mlx5_core_dev *mdev;
1388 struct mlx5_cqwq wq;
1389 struct mlx5_wq_ctrl wq_ctrl;
1390 struct mlx5_core_cq mcq;
1391 struct mlx5dr_qp *qp;
1392 };
1393
1394 struct mlx5dr_mr {
1395 struct mlx5_core_dev *mdev;
1396 u32 mkey;
1397 dma_addr_t dma_addr;
1398 void *addr;
1399 size_t size;
1400 };
1401
1402 #define MAX_SEND_CQE 64
1403 #define MIN_READ_SYNC 64
1404
1405 struct mlx5dr_send_ring {
1406 struct mlx5dr_cq *cq;
1407 struct mlx5dr_qp *qp;
1408 struct mlx5dr_mr *mr;
1409 /* How much wqes are waiting for completion */
1410 u32 pending_wqe;
1411 /* Signal request per this trash hold value */
1412 u16 signal_th;
1413 /* Each post_send_size less than max_post_send_size */
1414 u32 max_post_send_size;
1415 /* manage the send queue */
1416 u32 tx_head;
1417 void *buf;
1418 u32 buf_size;
1419 u8 sync_buff[MIN_READ_SYNC];
1420 struct mlx5dr_mr *sync_mr;
1421 spinlock_t lock; /* Protect the data path of the send ring */
1422 bool err_state; /* send_ring is not usable in err state */
1423 };
1424
1425 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn);
1426 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn,
1427 struct mlx5dr_send_ring *send_ring);
1428 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn);
1429 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn,
1430 struct mlx5dr_ste *ste,
1431 u8 *data,
1432 u16 size,
1433 u16 offset);
1434 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn,
1435 struct mlx5dr_ste_htbl *htbl,
1436 u8 *formatted_ste, u8 *mask);
1437 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn,
1438 struct mlx5dr_ste_htbl *htbl,
1439 u8 *ste_init_data,
1440 bool update_hw_ste);
1441 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn,
1442 struct mlx5dr_action *action);
1443
1444 int mlx5dr_send_info_pool_create(struct mlx5dr_domain *dmn);
1445 void mlx5dr_send_info_pool_destroy(struct mlx5dr_domain *dmn);
1446 struct mlx5dr_ste_send_info *mlx5dr_send_info_alloc(struct mlx5dr_domain *dmn,
1447 enum mlx5dr_domain_nic_type nic_type);
1448 void mlx5dr_send_info_free(struct mlx5dr_ste_send_info *ste_send_info);
1449
1450 struct mlx5dr_cmd_ft_info {
1451 u32 id;
1452 u16 vport;
1453 enum fs_flow_table_type type;
1454 };
1455
1456 struct mlx5dr_cmd_flow_destination_hw_info {
1457 enum mlx5_flow_destination_type type;
1458 union {
1459 u32 tir_num;
1460 u32 ft_num;
1461 u32 ft_id;
1462 u32 counter_id;
1463 u32 sampler_id;
1464 struct {
1465 u16 num;
1466 u16 vhca_id;
1467 u32 reformat_id;
1468 u8 flags;
1469 } vport;
1470 };
1471 };
1472
1473 struct mlx5dr_cmd_fte_info {
1474 u32 dests_size;
1475 u32 index;
1476 struct mlx5_flow_context flow_context;
1477 u32 *val;
1478 struct mlx5_flow_act action;
1479 struct mlx5dr_cmd_flow_destination_hw_info *dest_arr;
1480 bool ignore_flow_level;
1481 };
1482
1483 int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev,
1484 int opmod, int modify_mask,
1485 struct mlx5dr_cmd_ft_info *ft,
1486 u32 group_id,
1487 struct mlx5dr_cmd_fte_info *fte);
1488
1489 bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps);
1490
1491 struct mlx5dr_fw_recalc_cs_ft {
1492 u64 rx_icm_addr;
1493 u32 table_id;
1494 u32 group_id;
1495 u32 modify_hdr_id;
1496 };
1497
1498 struct mlx5dr_fw_recalc_cs_ft *
1499 mlx5dr_fw_create_recalc_cs_ft(struct mlx5dr_domain *dmn, u16 vport_num);
1500 void mlx5dr_fw_destroy_recalc_cs_ft(struct mlx5dr_domain *dmn,
1501 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft);
1502 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn,
1503 u16 vport_num,
1504 u64 *rx_icm_addr);
1505 int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn,
1506 struct mlx5dr_cmd_flow_destination_hw_info *dest,
1507 int num_dest,
1508 bool reformat_req,
1509 u32 *tbl_id,
1510 u32 *group_id,
1511 bool ignore_flow_level,
1512 u32 flow_source);
1513 void mlx5dr_fw_destroy_md_tbl(struct mlx5dr_domain *dmn, u32 tbl_id,
1514 u32 group_id);
1515
mlx5dr_is_fw_table(struct mlx5_flow_table * ft)1516 static inline bool mlx5dr_is_fw_table(struct mlx5_flow_table *ft)
1517 {
1518 return !ft->fs_dr_table.dr_table;
1519 }
1520
mlx5dr_supp_match_ranges(struct mlx5_core_dev * dev)1521 static inline bool mlx5dr_supp_match_ranges(struct mlx5_core_dev *dev)
1522 {
1523 return (MLX5_CAP_GEN(dev, steering_format_version) >=
1524 MLX5_STEERING_FORMAT_CONNECTX_6DX) &&
1525 (MLX5_CAP_GEN_64(dev, match_definer_format_supported) &
1526 (1ULL << MLX5_IFC_DEFINER_FORMAT_ID_SELECT));
1527 }
1528
1529 #endif /* _DR_TYPES_H_ */
1530