1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3
4 #include <net/macsec.h>
5 #include <linux/netdevice.h>
6 #include <linux/mlx5/qp.h>
7 #include "fs_core.h"
8 #include "en/fs.h"
9 #include "en_accel/macsec_fs.h"
10 #include "mlx5_core.h"
11
12 /* MACsec TX flow steering */
13 #define CRYPTO_NUM_MAXSEC_FTE BIT(15)
14 #define CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE 1
15
16 #define TX_CRYPTO_TABLE_LEVEL 0
17 #define TX_CRYPTO_TABLE_NUM_GROUPS 3
18 #define TX_CRYPTO_TABLE_MKE_GROUP_SIZE 1
19 #define TX_CRYPTO_TABLE_SA_GROUP_SIZE \
20 (CRYPTO_NUM_MAXSEC_FTE - (TX_CRYPTO_TABLE_MKE_GROUP_SIZE + \
21 CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE))
22 #define TX_CHECK_TABLE_LEVEL 1
23 #define TX_CHECK_TABLE_NUM_FTE 2
24 #define RX_CRYPTO_TABLE_LEVEL 0
25 #define RX_CHECK_TABLE_LEVEL 1
26 #define RX_CHECK_TABLE_NUM_FTE 3
27 #define RX_CRYPTO_TABLE_NUM_GROUPS 3
28 #define RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE \
29 ((CRYPTO_NUM_MAXSEC_FTE - CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE) / 2)
30 #define RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE \
31 (CRYPTO_NUM_MAXSEC_FTE - RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE)
32 #define RX_NUM_OF_RULES_PER_SA 2
33
34 #define MLX5_MACSEC_TAG_LEN 8 /* SecTAG length with ethertype and without the optional SCI */
35 #define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK 0x23
36 #define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET 0x8
37 #define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET 0x5
38 #define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT (0x1 << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET)
39 #define MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI 0x8
40 #define MLX5_SECTAG_HEADER_SIZE_WITH_SCI (MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI + MACSEC_SCI_LEN)
41
42 /* MACsec RX flow steering */
43 #define MLX5_ETH_WQE_FT_META_MACSEC_MASK 0x3E
44
45 struct mlx5_sectag_header {
46 __be16 ethertype;
47 u8 tci_an;
48 u8 sl;
49 u32 pn;
50 u8 sci[MACSEC_SCI_LEN]; /* optional */
51 } __packed;
52
53 struct mlx5e_macsec_tx_rule {
54 struct mlx5_flow_handle *rule;
55 struct mlx5_pkt_reformat *pkt_reformat;
56 u32 fs_id;
57 };
58
59 struct mlx5e_macsec_tables {
60 struct mlx5e_flow_table ft_crypto;
61 struct mlx5_flow_handle *crypto_miss_rule;
62
63 struct mlx5_flow_table *ft_check;
64 struct mlx5_flow_group *ft_check_group;
65 struct mlx5_fc *check_miss_rule_counter;
66 struct mlx5_flow_handle *check_miss_rule;
67 struct mlx5_fc *check_rule_counter;
68
69 u32 refcnt;
70 };
71
72 struct mlx5e_macsec_tx {
73 struct mlx5_flow_handle *crypto_mke_rule;
74 struct mlx5_flow_handle *check_rule;
75
76 struct ida tx_halloc;
77
78 struct mlx5e_macsec_tables tables;
79 };
80
81 struct mlx5e_macsec_rx_rule {
82 struct mlx5_flow_handle *rule[RX_NUM_OF_RULES_PER_SA];
83 struct mlx5_modify_hdr *meta_modhdr;
84 };
85
86 struct mlx5e_macsec_rx {
87 struct mlx5_flow_handle *check_rule[2];
88 struct mlx5_pkt_reformat *check_rule_pkt_reformat[2];
89
90 struct mlx5e_macsec_tables tables;
91 };
92
93 union mlx5e_macsec_rule {
94 struct mlx5e_macsec_tx_rule tx_rule;
95 struct mlx5e_macsec_rx_rule rx_rule;
96 };
97
98 struct mlx5e_macsec_fs {
99 struct mlx5_core_dev *mdev;
100 struct net_device *netdev;
101 struct mlx5e_macsec_tx *tx_fs;
102 struct mlx5e_macsec_rx *rx_fs;
103 };
104
macsec_fs_tx_destroy(struct mlx5e_macsec_fs * macsec_fs)105 static void macsec_fs_tx_destroy(struct mlx5e_macsec_fs *macsec_fs)
106 {
107 struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
108 struct mlx5e_macsec_tables *tx_tables;
109
110 tx_tables = &tx_fs->tables;
111
112 /* Tx check table */
113 if (tx_fs->check_rule) {
114 mlx5_del_flow_rules(tx_fs->check_rule);
115 tx_fs->check_rule = NULL;
116 }
117
118 if (tx_tables->check_miss_rule) {
119 mlx5_del_flow_rules(tx_tables->check_miss_rule);
120 tx_tables->check_miss_rule = NULL;
121 }
122
123 if (tx_tables->ft_check_group) {
124 mlx5_destroy_flow_group(tx_tables->ft_check_group);
125 tx_tables->ft_check_group = NULL;
126 }
127
128 if (tx_tables->ft_check) {
129 mlx5_destroy_flow_table(tx_tables->ft_check);
130 tx_tables->ft_check = NULL;
131 }
132
133 /* Tx crypto table */
134 if (tx_fs->crypto_mke_rule) {
135 mlx5_del_flow_rules(tx_fs->crypto_mke_rule);
136 tx_fs->crypto_mke_rule = NULL;
137 }
138
139 if (tx_tables->crypto_miss_rule) {
140 mlx5_del_flow_rules(tx_tables->crypto_miss_rule);
141 tx_tables->crypto_miss_rule = NULL;
142 }
143
144 mlx5e_destroy_flow_table(&tx_tables->ft_crypto);
145 }
146
macsec_fs_tx_create_crypto_table_groups(struct mlx5e_flow_table * ft)147 static int macsec_fs_tx_create_crypto_table_groups(struct mlx5e_flow_table *ft)
148 {
149 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
150 int mclen = MLX5_ST_SZ_BYTES(fte_match_param);
151 int ix = 0;
152 u32 *in;
153 int err;
154 u8 *mc;
155
156 ft->g = kcalloc(TX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
157 if (!ft->g)
158 return -ENOMEM;
159 in = kvzalloc(inlen, GFP_KERNEL);
160
161 if (!in) {
162 kfree(ft->g);
163 return -ENOMEM;
164 }
165
166 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
167
168 /* Flow Group for MKE match */
169 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
170 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
171
172 MLX5_SET_CFG(in, start_flow_index, ix);
173 ix += TX_CRYPTO_TABLE_MKE_GROUP_SIZE;
174 MLX5_SET_CFG(in, end_flow_index, ix - 1);
175 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
176 if (IS_ERR(ft->g[ft->num_groups]))
177 goto err;
178 ft->num_groups++;
179
180 /* Flow Group for SA rules */
181 memset(in, 0, inlen);
182 memset(mc, 0, mclen);
183 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2);
184 MLX5_SET(fte_match_param, mc, misc_parameters_2.metadata_reg_a,
185 MLX5_ETH_WQE_FT_META_MACSEC_MASK);
186
187 MLX5_SET_CFG(in, start_flow_index, ix);
188 ix += TX_CRYPTO_TABLE_SA_GROUP_SIZE;
189 MLX5_SET_CFG(in, end_flow_index, ix - 1);
190 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
191 if (IS_ERR(ft->g[ft->num_groups]))
192 goto err;
193 ft->num_groups++;
194
195 /* Flow Group for l2 traps */
196 memset(in, 0, inlen);
197 memset(mc, 0, mclen);
198 MLX5_SET_CFG(in, start_flow_index, ix);
199 ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE;
200 MLX5_SET_CFG(in, end_flow_index, ix - 1);
201 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
202 if (IS_ERR(ft->g[ft->num_groups]))
203 goto err;
204 ft->num_groups++;
205
206 kvfree(in);
207 return 0;
208
209 err:
210 err = PTR_ERR(ft->g[ft->num_groups]);
211 ft->g[ft->num_groups] = NULL;
212 kvfree(in);
213
214 return err;
215 }
216
217 static struct mlx5_flow_table
macsec_fs_auto_group_table_create(struct mlx5_flow_namespace * ns,int flags,int level,int max_fte)218 *macsec_fs_auto_group_table_create(struct mlx5_flow_namespace *ns, int flags,
219 int level, int max_fte)
220 {
221 struct mlx5_flow_table_attr ft_attr = {};
222 struct mlx5_flow_table *fdb = NULL;
223
224 /* reserve entry for the match all miss group and rule */
225 ft_attr.autogroup.num_reserved_entries = 1;
226 ft_attr.autogroup.max_num_groups = 1;
227 ft_attr.prio = 0;
228 ft_attr.flags = flags;
229 ft_attr.level = level;
230 ft_attr.max_fte = max_fte;
231
232 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
233
234 return fdb;
235 }
236
macsec_fs_tx_create(struct mlx5e_macsec_fs * macsec_fs)237 static int macsec_fs_tx_create(struct mlx5e_macsec_fs *macsec_fs)
238 {
239 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
240 struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
241 struct net_device *netdev = macsec_fs->netdev;
242 struct mlx5_flow_table_attr ft_attr = {};
243 struct mlx5_flow_destination dest = {};
244 struct mlx5e_macsec_tables *tx_tables;
245 struct mlx5_flow_act flow_act = {};
246 struct mlx5e_flow_table *ft_crypto;
247 struct mlx5_flow_table *flow_table;
248 struct mlx5_flow_group *flow_group;
249 struct mlx5_flow_namespace *ns;
250 struct mlx5_flow_handle *rule;
251 struct mlx5_flow_spec *spec;
252 u32 *flow_group_in;
253 int err;
254
255 ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_EGRESS_MACSEC);
256 if (!ns)
257 return -ENOMEM;
258
259 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
260 if (!spec)
261 return -ENOMEM;
262
263 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
264 if (!flow_group_in) {
265 err = -ENOMEM;
266 goto out_spec;
267 }
268
269 tx_tables = &tx_fs->tables;
270 ft_crypto = &tx_tables->ft_crypto;
271
272 /* Tx crypto table */
273 ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
274 ft_attr.level = TX_CRYPTO_TABLE_LEVEL;
275 ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE;
276
277 flow_table = mlx5_create_flow_table(ns, &ft_attr);
278 if (IS_ERR(flow_table)) {
279 err = PTR_ERR(flow_table);
280 netdev_err(netdev, "Failed to create MACsec Tx crypto table err(%d)\n", err);
281 goto out_flow_group;
282 }
283 ft_crypto->t = flow_table;
284
285 /* Tx crypto table groups */
286 err = macsec_fs_tx_create_crypto_table_groups(ft_crypto);
287 if (err) {
288 netdev_err(netdev,
289 "Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
290 err);
291 goto err;
292 }
293
294 /* Tx crypto table MKE rule - MKE packets shouldn't be offloaded */
295 memset(&flow_act, 0, sizeof(flow_act));
296 memset(spec, 0, sizeof(*spec));
297 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
298
299 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
300 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_PAE);
301 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW;
302
303 rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, NULL, 0);
304 if (IS_ERR(rule)) {
305 err = PTR_ERR(rule);
306 netdev_err(netdev, "Failed to add MACsec TX MKE rule, err=%d\n", err);
307 goto err;
308 }
309 tx_fs->crypto_mke_rule = rule;
310
311 /* Tx crypto table Default miss rule */
312 memset(&flow_act, 0, sizeof(flow_act));
313 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW;
314 rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0);
315 if (IS_ERR(rule)) {
316 err = PTR_ERR(rule);
317 netdev_err(netdev, "Failed to add MACsec Tx table default miss rule %d\n", err);
318 goto err;
319 }
320 tx_tables->crypto_miss_rule = rule;
321
322 /* Tx check table */
323 flow_table = macsec_fs_auto_group_table_create(ns, 0, TX_CHECK_TABLE_LEVEL,
324 TX_CHECK_TABLE_NUM_FTE);
325 if (IS_ERR(flow_table)) {
326 err = PTR_ERR(flow_table);
327 netdev_err(netdev, "fail to create MACsec TX check table, err(%d)\n", err);
328 goto err;
329 }
330 tx_tables->ft_check = flow_table;
331
332 /* Tx check table Default miss group/rule */
333 memset(flow_group_in, 0, inlen);
334 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1);
335 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1);
336 flow_group = mlx5_create_flow_group(tx_tables->ft_check, flow_group_in);
337 if (IS_ERR(flow_group)) {
338 err = PTR_ERR(flow_group);
339 netdev_err(netdev,
340 "Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
341 err);
342 goto err;
343 }
344 tx_tables->ft_check_group = flow_group;
345
346 /* Tx check table default drop rule */
347 memset(&dest, 0, sizeof(struct mlx5_flow_destination));
348 memset(&flow_act, 0, sizeof(flow_act));
349 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
350 dest.counter_id = mlx5_fc_id(tx_tables->check_miss_rule_counter);
351 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
352 rule = mlx5_add_flow_rules(tx_tables->ft_check, NULL, &flow_act, &dest, 1);
353 if (IS_ERR(rule)) {
354 err = PTR_ERR(rule);
355 netdev_err(netdev, "Failed to added MACsec tx check drop rule, err(%d)\n", err);
356 goto err;
357 }
358 tx_tables->check_miss_rule = rule;
359
360 /* Tx check table rule */
361 memset(spec, 0, sizeof(struct mlx5_flow_spec));
362 memset(&dest, 0, sizeof(struct mlx5_flow_destination));
363 memset(&flow_act, 0, sizeof(flow_act));
364
365 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4);
366 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0);
367 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
368
369 flow_act.flags = FLOW_ACT_NO_APPEND;
370 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_COUNT;
371 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
372 dest.counter_id = mlx5_fc_id(tx_tables->check_rule_counter);
373 rule = mlx5_add_flow_rules(tx_tables->ft_check, spec, &flow_act, &dest, 1);
374 if (IS_ERR(rule)) {
375 err = PTR_ERR(rule);
376 netdev_err(netdev, "Failed to add MACsec check rule, err=%d\n", err);
377 goto err;
378 }
379 tx_fs->check_rule = rule;
380
381 goto out_flow_group;
382
383 err:
384 macsec_fs_tx_destroy(macsec_fs);
385 out_flow_group:
386 kvfree(flow_group_in);
387 out_spec:
388 kvfree(spec);
389 return err;
390 }
391
macsec_fs_tx_ft_get(struct mlx5e_macsec_fs * macsec_fs)392 static int macsec_fs_tx_ft_get(struct mlx5e_macsec_fs *macsec_fs)
393 {
394 struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
395 struct mlx5e_macsec_tables *tx_tables;
396 int err = 0;
397
398 tx_tables = &tx_fs->tables;
399 if (tx_tables->refcnt)
400 goto out;
401
402 err = macsec_fs_tx_create(macsec_fs);
403 if (err)
404 return err;
405
406 out:
407 tx_tables->refcnt++;
408 return err;
409 }
410
macsec_fs_tx_ft_put(struct mlx5e_macsec_fs * macsec_fs)411 static void macsec_fs_tx_ft_put(struct mlx5e_macsec_fs *macsec_fs)
412 {
413 struct mlx5e_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables;
414
415 if (--tx_tables->refcnt)
416 return;
417
418 macsec_fs_tx_destroy(macsec_fs);
419 }
420
macsec_fs_tx_setup_fte(struct mlx5e_macsec_fs * macsec_fs,struct mlx5_flow_spec * spec,struct mlx5_flow_act * flow_act,u32 macsec_obj_id,u32 * fs_id)421 static int macsec_fs_tx_setup_fte(struct mlx5e_macsec_fs *macsec_fs,
422 struct mlx5_flow_spec *spec,
423 struct mlx5_flow_act *flow_act,
424 u32 macsec_obj_id,
425 u32 *fs_id)
426 {
427 struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
428 int err = 0;
429 u32 id;
430
431 err = ida_alloc_range(&tx_fs->tx_halloc, 1,
432 MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES,
433 GFP_KERNEL);
434 if (err < 0)
435 return err;
436
437 id = err;
438 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
439
440 /* Metadata match */
441 MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a,
442 MLX5_ETH_WQE_FT_META_MACSEC_MASK);
443 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a,
444 MLX5_ETH_WQE_FT_META_MACSEC | id << 2);
445
446 *fs_id = id;
447 flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC;
448 flow_act->crypto.obj_id = macsec_obj_id;
449
450 mlx5_core_dbg(macsec_fs->mdev, "Tx fte: macsec obj_id %u, fs_id %u\n", macsec_obj_id, id);
451 return 0;
452 }
453
macsec_fs_tx_create_sectag_header(const struct macsec_context * ctx,char * reformatbf,size_t * reformat_size)454 static void macsec_fs_tx_create_sectag_header(const struct macsec_context *ctx,
455 char *reformatbf,
456 size_t *reformat_size)
457 {
458 const struct macsec_secy *secy = ctx->secy;
459 bool sci_present = macsec_send_sci(secy);
460 struct mlx5_sectag_header sectag = {};
461 const struct macsec_tx_sc *tx_sc;
462
463 tx_sc = &secy->tx_sc;
464 sectag.ethertype = htons(ETH_P_MACSEC);
465
466 if (sci_present) {
467 sectag.tci_an |= MACSEC_TCI_SC;
468 memcpy(§ag.sci, &secy->sci,
469 sizeof(sectag.sci));
470 } else {
471 if (tx_sc->end_station)
472 sectag.tci_an |= MACSEC_TCI_ES;
473 if (tx_sc->scb)
474 sectag.tci_an |= MACSEC_TCI_SCB;
475 }
476
477 /* With GCM, C/E clear for !encrypt, both set for encrypt */
478 if (tx_sc->encrypt)
479 sectag.tci_an |= MACSEC_TCI_CONFID;
480 else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
481 sectag.tci_an |= MACSEC_TCI_C;
482
483 sectag.tci_an |= tx_sc->encoding_sa;
484
485 *reformat_size = MLX5_MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0);
486
487 memcpy(reformatbf, §ag, *reformat_size);
488 }
489
macsec_fs_tx_del_rule(struct mlx5e_macsec_fs * macsec_fs,struct mlx5e_macsec_tx_rule * tx_rule)490 static void macsec_fs_tx_del_rule(struct mlx5e_macsec_fs *macsec_fs,
491 struct mlx5e_macsec_tx_rule *tx_rule)
492 {
493 if (tx_rule->rule) {
494 mlx5_del_flow_rules(tx_rule->rule);
495 tx_rule->rule = NULL;
496 }
497
498 if (tx_rule->pkt_reformat) {
499 mlx5_packet_reformat_dealloc(macsec_fs->mdev, tx_rule->pkt_reformat);
500 tx_rule->pkt_reformat = NULL;
501 }
502
503 if (tx_rule->fs_id) {
504 ida_free(&macsec_fs->tx_fs->tx_halloc, tx_rule->fs_id);
505 tx_rule->fs_id = 0;
506 }
507
508 kfree(tx_rule);
509
510 macsec_fs_tx_ft_put(macsec_fs);
511 }
512
513 static union mlx5e_macsec_rule *
macsec_fs_tx_add_rule(struct mlx5e_macsec_fs * macsec_fs,const struct macsec_context * macsec_ctx,struct mlx5_macsec_rule_attrs * attrs,u32 * sa_fs_id)514 macsec_fs_tx_add_rule(struct mlx5e_macsec_fs *macsec_fs,
515 const struct macsec_context *macsec_ctx,
516 struct mlx5_macsec_rule_attrs *attrs,
517 u32 *sa_fs_id)
518 {
519 char reformatbf[MLX5_MACSEC_TAG_LEN + MACSEC_SCI_LEN];
520 struct mlx5_pkt_reformat_params reformat_params = {};
521 struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
522 struct net_device *netdev = macsec_fs->netdev;
523 union mlx5e_macsec_rule *macsec_rule = NULL;
524 struct mlx5_flow_destination dest = {};
525 struct mlx5e_macsec_tables *tx_tables;
526 struct mlx5e_macsec_tx_rule *tx_rule;
527 struct mlx5_flow_act flow_act = {};
528 struct mlx5_flow_handle *rule;
529 struct mlx5_flow_spec *spec;
530 size_t reformat_size;
531 int err = 0;
532 u32 fs_id;
533
534 tx_tables = &tx_fs->tables;
535
536 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
537 if (!spec)
538 return NULL;
539
540 err = macsec_fs_tx_ft_get(macsec_fs);
541 if (err)
542 goto out_spec;
543
544 macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL);
545 if (!macsec_rule) {
546 macsec_fs_tx_ft_put(macsec_fs);
547 goto out_spec;
548 }
549
550 tx_rule = &macsec_rule->tx_rule;
551
552 /* Tx crypto table crypto rule */
553 macsec_fs_tx_create_sectag_header(macsec_ctx, reformatbf, &reformat_size);
554
555 reformat_params.type = MLX5_REFORMAT_TYPE_ADD_MACSEC;
556 reformat_params.size = reformat_size;
557 reformat_params.data = reformatbf;
558 flow_act.pkt_reformat = mlx5_packet_reformat_alloc(macsec_fs->mdev,
559 &reformat_params,
560 MLX5_FLOW_NAMESPACE_EGRESS_MACSEC);
561 if (IS_ERR(flow_act.pkt_reformat)) {
562 err = PTR_ERR(flow_act.pkt_reformat);
563 netdev_err(netdev, "Failed to allocate MACsec Tx reformat context err=%d\n", err);
564 goto err;
565 }
566 tx_rule->pkt_reformat = flow_act.pkt_reformat;
567
568 err = macsec_fs_tx_setup_fte(macsec_fs, spec, &flow_act, attrs->macsec_obj_id, &fs_id);
569 if (err) {
570 netdev_err(netdev,
571 "Failed to add packet reformat for MACsec TX crypto rule, err=%d\n",
572 err);
573 goto err;
574 }
575
576 tx_rule->fs_id = fs_id;
577 *sa_fs_id = fs_id;
578
579 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
580 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
581 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
582 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
583 dest.ft = tx_tables->ft_check;
584 rule = mlx5_add_flow_rules(tx_tables->ft_crypto.t, spec, &flow_act, &dest, 1);
585 if (IS_ERR(rule)) {
586 err = PTR_ERR(rule);
587 netdev_err(netdev, "Failed to add MACsec TX crypto rule, err=%d\n", err);
588 goto err;
589 }
590 tx_rule->rule = rule;
591
592 goto out_spec;
593
594 err:
595 macsec_fs_tx_del_rule(macsec_fs, tx_rule);
596 macsec_rule = NULL;
597 out_spec:
598 kvfree(spec);
599
600 return macsec_rule;
601 }
602
macsec_fs_tx_cleanup(struct mlx5e_macsec_fs * macsec_fs)603 static void macsec_fs_tx_cleanup(struct mlx5e_macsec_fs *macsec_fs)
604 {
605 struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
606 struct mlx5_core_dev *mdev = macsec_fs->mdev;
607 struct mlx5e_macsec_tables *tx_tables;
608
609 if (!tx_fs)
610 return;
611
612 tx_tables = &tx_fs->tables;
613 if (tx_tables->refcnt) {
614 netdev_err(macsec_fs->netdev,
615 "Can't destroy MACsec offload tx_fs, refcnt(%u) isn't 0\n",
616 tx_tables->refcnt);
617 return;
618 }
619
620 ida_destroy(&tx_fs->tx_halloc);
621
622 if (tx_tables->check_miss_rule_counter) {
623 mlx5_fc_destroy(mdev, tx_tables->check_miss_rule_counter);
624 tx_tables->check_miss_rule_counter = NULL;
625 }
626
627 if (tx_tables->check_rule_counter) {
628 mlx5_fc_destroy(mdev, tx_tables->check_rule_counter);
629 tx_tables->check_rule_counter = NULL;
630 }
631
632 kfree(tx_fs);
633 macsec_fs->tx_fs = NULL;
634 }
635
macsec_fs_tx_init(struct mlx5e_macsec_fs * macsec_fs)636 static int macsec_fs_tx_init(struct mlx5e_macsec_fs *macsec_fs)
637 {
638 struct net_device *netdev = macsec_fs->netdev;
639 struct mlx5_core_dev *mdev = macsec_fs->mdev;
640 struct mlx5e_macsec_tables *tx_tables;
641 struct mlx5e_macsec_tx *tx_fs;
642 struct mlx5_fc *flow_counter;
643 int err;
644
645 tx_fs = kzalloc(sizeof(*tx_fs), GFP_KERNEL);
646 if (!tx_fs)
647 return -ENOMEM;
648
649 tx_tables = &tx_fs->tables;
650
651 flow_counter = mlx5_fc_create(mdev, false);
652 if (IS_ERR(flow_counter)) {
653 err = PTR_ERR(flow_counter);
654 netdev_err(netdev,
655 "Failed to create MACsec Tx encrypt flow counter, err(%d)\n",
656 err);
657 goto err_encrypt_counter;
658 }
659 tx_tables->check_rule_counter = flow_counter;
660
661 flow_counter = mlx5_fc_create(mdev, false);
662 if (IS_ERR(flow_counter)) {
663 err = PTR_ERR(flow_counter);
664 netdev_err(netdev,
665 "Failed to create MACsec Tx drop flow counter, err(%d)\n",
666 err);
667 goto err_drop_counter;
668 }
669 tx_tables->check_miss_rule_counter = flow_counter;
670
671 ida_init(&tx_fs->tx_halloc);
672
673 macsec_fs->tx_fs = tx_fs;
674
675 return 0;
676
677 err_drop_counter:
678 mlx5_fc_destroy(mdev, tx_tables->check_rule_counter);
679 tx_tables->check_rule_counter = NULL;
680
681 err_encrypt_counter:
682 kfree(tx_fs);
683 macsec_fs->tx_fs = NULL;
684
685 return err;
686 }
687
macsec_fs_rx_destroy(struct mlx5e_macsec_fs * macsec_fs)688 static void macsec_fs_rx_destroy(struct mlx5e_macsec_fs *macsec_fs)
689 {
690 struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
691 struct mlx5e_macsec_tables *rx_tables;
692 int i;
693
694 /* Rx check table */
695 for (i = 1; i >= 0; --i) {
696 if (rx_fs->check_rule[i]) {
697 mlx5_del_flow_rules(rx_fs->check_rule[i]);
698 rx_fs->check_rule[i] = NULL;
699 }
700
701 if (rx_fs->check_rule_pkt_reformat[i]) {
702 mlx5_packet_reformat_dealloc(macsec_fs->mdev,
703 rx_fs->check_rule_pkt_reformat[i]);
704 rx_fs->check_rule_pkt_reformat[i] = NULL;
705 }
706 }
707
708 rx_tables = &rx_fs->tables;
709
710 if (rx_tables->check_miss_rule) {
711 mlx5_del_flow_rules(rx_tables->check_miss_rule);
712 rx_tables->check_miss_rule = NULL;
713 }
714
715 if (rx_tables->ft_check_group) {
716 mlx5_destroy_flow_group(rx_tables->ft_check_group);
717 rx_tables->ft_check_group = NULL;
718 }
719
720 if (rx_tables->ft_check) {
721 mlx5_destroy_flow_table(rx_tables->ft_check);
722 rx_tables->ft_check = NULL;
723 }
724
725 /* Rx crypto table */
726 if (rx_tables->crypto_miss_rule) {
727 mlx5_del_flow_rules(rx_tables->crypto_miss_rule);
728 rx_tables->crypto_miss_rule = NULL;
729 }
730
731 mlx5e_destroy_flow_table(&rx_tables->ft_crypto);
732 }
733
macsec_fs_rx_create_crypto_table_groups(struct mlx5e_flow_table * ft)734 static int macsec_fs_rx_create_crypto_table_groups(struct mlx5e_flow_table *ft)
735 {
736 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
737 int mclen = MLX5_ST_SZ_BYTES(fte_match_param);
738 int ix = 0;
739 u32 *in;
740 int err;
741 u8 *mc;
742
743 ft->g = kcalloc(RX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
744 if (!ft->g)
745 return -ENOMEM;
746
747 in = kvzalloc(inlen, GFP_KERNEL);
748 if (!in) {
749 kfree(ft->g);
750 return -ENOMEM;
751 }
752
753 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
754
755 /* Flow group for SA rule with SCI */
756 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS |
757 MLX5_MATCH_MISC_PARAMETERS_5);
758 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
759
760 MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0,
761 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK <<
762 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
763 MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_2);
764 MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_3);
765
766 MLX5_SET_CFG(in, start_flow_index, ix);
767 ix += RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE;
768 MLX5_SET_CFG(in, end_flow_index, ix - 1);
769 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
770 if (IS_ERR(ft->g[ft->num_groups]))
771 goto err;
772 ft->num_groups++;
773
774 /* Flow group for SA rule without SCI */
775 memset(in, 0, inlen);
776 memset(mc, 0, mclen);
777 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS |
778 MLX5_MATCH_MISC_PARAMETERS_5);
779 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_47_16);
780 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_15_0);
781 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
782
783 MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0,
784 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
785
786 MLX5_SET_CFG(in, start_flow_index, ix);
787 ix += RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE;
788 MLX5_SET_CFG(in, end_flow_index, ix - 1);
789 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
790 if (IS_ERR(ft->g[ft->num_groups]))
791 goto err;
792 ft->num_groups++;
793
794 /* Flow Group for l2 traps */
795 memset(in, 0, inlen);
796 memset(mc, 0, mclen);
797 MLX5_SET_CFG(in, start_flow_index, ix);
798 ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE;
799 MLX5_SET_CFG(in, end_flow_index, ix - 1);
800 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
801 if (IS_ERR(ft->g[ft->num_groups]))
802 goto err;
803 ft->num_groups++;
804
805 kvfree(in);
806 return 0;
807
808 err:
809 err = PTR_ERR(ft->g[ft->num_groups]);
810 ft->g[ft->num_groups] = NULL;
811 kvfree(in);
812
813 return err;
814 }
815
macsec_fs_rx_create_check_decap_rule(struct mlx5e_macsec_fs * macsec_fs,struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_flow_spec * spec,int reformat_param_size)816 static int macsec_fs_rx_create_check_decap_rule(struct mlx5e_macsec_fs *macsec_fs,
817 struct mlx5_flow_destination *dest,
818 struct mlx5_flow_act *flow_act,
819 struct mlx5_flow_spec *spec,
820 int reformat_param_size)
821 {
822 int rule_index = (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) ? 0 : 1;
823 u8 mlx5_reformat_buf[MLX5_SECTAG_HEADER_SIZE_WITH_SCI];
824 struct mlx5_pkt_reformat_params reformat_params = {};
825 struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
826 struct net_device *netdev = macsec_fs->netdev;
827 struct mlx5e_macsec_tables *rx_tables;
828 struct mlx5_flow_handle *rule;
829 int err = 0;
830
831 rx_tables = &rx_fs->tables;
832
833 /* Rx check table decap 16B rule */
834 memset(dest, 0, sizeof(*dest));
835 memset(flow_act, 0, sizeof(*flow_act));
836 memset(spec, 0, sizeof(*spec));
837
838 reformat_params.type = MLX5_REFORMAT_TYPE_DEL_MACSEC;
839 reformat_params.size = reformat_param_size;
840 reformat_params.data = mlx5_reformat_buf;
841 flow_act->pkt_reformat = mlx5_packet_reformat_alloc(macsec_fs->mdev,
842 &reformat_params,
843 MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC);
844 if (IS_ERR(flow_act->pkt_reformat)) {
845 err = PTR_ERR(flow_act->pkt_reformat);
846 netdev_err(netdev, "Failed to allocate MACsec Rx reformat context err=%d\n", err);
847 return err;
848 }
849 rx_fs->check_rule_pkt_reformat[rule_index] = flow_act->pkt_reformat;
850
851 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
852 /* MACsec syndrome match */
853 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.macsec_syndrome);
854 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.macsec_syndrome, 0);
855 /* ASO return reg syndrome match */
856 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4);
857 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0);
858
859 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5;
860 /* Sectag TCI SC present bit*/
861 MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0,
862 MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
863
864 if (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI)
865 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0,
866 MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT <<
867 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
868
869 flow_act->flags = FLOW_ACT_NO_APPEND;
870 flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO |
871 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT |
872 MLX5_FLOW_CONTEXT_ACTION_COUNT;
873 dest->type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
874 dest->counter_id = mlx5_fc_id(rx_tables->check_rule_counter);
875 rule = mlx5_add_flow_rules(rx_tables->ft_check, spec, flow_act, dest, 1);
876 if (IS_ERR(rule)) {
877 err = PTR_ERR(rule);
878 netdev_err(netdev, "Failed to add MACsec Rx check rule, err=%d\n", err);
879 return err;
880 }
881
882 rx_fs->check_rule[rule_index] = rule;
883
884 return 0;
885 }
886
macsec_fs_rx_create(struct mlx5e_macsec_fs * macsec_fs)887 static int macsec_fs_rx_create(struct mlx5e_macsec_fs *macsec_fs)
888 {
889 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
890 struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
891 struct net_device *netdev = macsec_fs->netdev;
892 struct mlx5_flow_table_attr ft_attr = {};
893 struct mlx5_flow_destination dest = {};
894 struct mlx5e_macsec_tables *rx_tables;
895 struct mlx5e_flow_table *ft_crypto;
896 struct mlx5_flow_table *flow_table;
897 struct mlx5_flow_group *flow_group;
898 struct mlx5_flow_act flow_act = {};
899 struct mlx5_flow_namespace *ns;
900 struct mlx5_flow_handle *rule;
901 struct mlx5_flow_spec *spec;
902 u32 *flow_group_in;
903 int err;
904
905 ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC);
906 if (!ns)
907 return -ENOMEM;
908
909 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
910 if (!spec)
911 return -ENOMEM;
912
913 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
914 if (!flow_group_in) {
915 err = -ENOMEM;
916 goto free_spec;
917 }
918
919 rx_tables = &rx_fs->tables;
920 ft_crypto = &rx_tables->ft_crypto;
921
922 /* Rx crypto table */
923 ft_attr.level = RX_CRYPTO_TABLE_LEVEL;
924 ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE;
925
926 flow_table = mlx5_create_flow_table(ns, &ft_attr);
927 if (IS_ERR(flow_table)) {
928 err = PTR_ERR(flow_table);
929 netdev_err(netdev, "Failed to create MACsec Rx crypto table err(%d)\n", err);
930 goto out_flow_group;
931 }
932 ft_crypto->t = flow_table;
933
934 /* Rx crypto table groups */
935 err = macsec_fs_rx_create_crypto_table_groups(ft_crypto);
936 if (err) {
937 netdev_err(netdev,
938 "Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
939 err);
940 goto err;
941 }
942
943 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
944 rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0);
945 if (IS_ERR(rule)) {
946 err = PTR_ERR(rule);
947 netdev_err(netdev,
948 "Failed to add MACsec Rx crypto table default miss rule %d\n",
949 err);
950 goto err;
951 }
952 rx_tables->crypto_miss_rule = rule;
953
954 /* Rx check table */
955 flow_table = macsec_fs_auto_group_table_create(ns,
956 MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT,
957 RX_CHECK_TABLE_LEVEL,
958 RX_CHECK_TABLE_NUM_FTE);
959 if (IS_ERR(flow_table)) {
960 err = PTR_ERR(flow_table);
961 netdev_err(netdev, "fail to create MACsec RX check table, err(%d)\n", err);
962 goto err;
963 }
964 rx_tables->ft_check = flow_table;
965
966 /* Rx check table Default miss group/rule */
967 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1);
968 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1);
969 flow_group = mlx5_create_flow_group(rx_tables->ft_check, flow_group_in);
970 if (IS_ERR(flow_group)) {
971 err = PTR_ERR(flow_group);
972 netdev_err(netdev,
973 "Failed to create default flow group for MACsec Rx check table err(%d)\n",
974 err);
975 goto err;
976 }
977 rx_tables->ft_check_group = flow_group;
978
979 /* Rx check table default drop rule */
980 memset(&flow_act, 0, sizeof(flow_act));
981
982 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
983 dest.counter_id = mlx5_fc_id(rx_tables->check_miss_rule_counter);
984 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
985 rule = mlx5_add_flow_rules(rx_tables->ft_check, NULL, &flow_act, &dest, 1);
986 if (IS_ERR(rule)) {
987 err = PTR_ERR(rule);
988 netdev_err(netdev, "Failed to added MACsec Rx check drop rule, err(%d)\n", err);
989 goto err;
990 }
991 rx_tables->check_miss_rule = rule;
992
993 /* Rx check table decap rules */
994 err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec,
995 MLX5_SECTAG_HEADER_SIZE_WITH_SCI);
996 if (err)
997 goto err;
998
999 err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec,
1000 MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI);
1001 if (err)
1002 goto err;
1003
1004 goto out_flow_group;
1005
1006 err:
1007 macsec_fs_rx_destroy(macsec_fs);
1008 out_flow_group:
1009 kvfree(flow_group_in);
1010 free_spec:
1011 kvfree(spec);
1012 return err;
1013 }
1014
macsec_fs_rx_ft_get(struct mlx5e_macsec_fs * macsec_fs)1015 static int macsec_fs_rx_ft_get(struct mlx5e_macsec_fs *macsec_fs)
1016 {
1017 struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
1018 int err = 0;
1019
1020 if (rx_tables->refcnt)
1021 goto out;
1022
1023 err = macsec_fs_rx_create(macsec_fs);
1024 if (err)
1025 return err;
1026
1027 out:
1028 rx_tables->refcnt++;
1029 return err;
1030 }
1031
macsec_fs_rx_ft_put(struct mlx5e_macsec_fs * macsec_fs)1032 static void macsec_fs_rx_ft_put(struct mlx5e_macsec_fs *macsec_fs)
1033 {
1034 struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
1035
1036 if (--rx_tables->refcnt)
1037 return;
1038
1039 macsec_fs_rx_destroy(macsec_fs);
1040 }
1041
macsec_fs_rx_del_rule(struct mlx5e_macsec_fs * macsec_fs,struct mlx5e_macsec_rx_rule * rx_rule)1042 static void macsec_fs_rx_del_rule(struct mlx5e_macsec_fs *macsec_fs,
1043 struct mlx5e_macsec_rx_rule *rx_rule)
1044 {
1045 int i;
1046
1047 for (i = 0; i < RX_NUM_OF_RULES_PER_SA; ++i) {
1048 if (rx_rule->rule[i]) {
1049 mlx5_del_flow_rules(rx_rule->rule[i]);
1050 rx_rule->rule[i] = NULL;
1051 }
1052 }
1053
1054 if (rx_rule->meta_modhdr) {
1055 mlx5_modify_header_dealloc(macsec_fs->mdev, rx_rule->meta_modhdr);
1056 rx_rule->meta_modhdr = NULL;
1057 }
1058
1059 kfree(rx_rule);
1060
1061 macsec_fs_rx_ft_put(macsec_fs);
1062 }
1063
macsec_fs_rx_setup_fte(struct mlx5_flow_spec * spec,struct mlx5_flow_act * flow_act,struct mlx5_macsec_rule_attrs * attrs,bool sci_present)1064 static void macsec_fs_rx_setup_fte(struct mlx5_flow_spec *spec,
1065 struct mlx5_flow_act *flow_act,
1066 struct mlx5_macsec_rule_attrs *attrs,
1067 bool sci_present)
1068 {
1069 u8 tci_an = (sci_present << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) | attrs->assoc_num;
1070 struct mlx5_flow_act_crypto_params *crypto_params = &flow_act->crypto;
1071 __be32 *sci_p = (__be32 *)(&attrs->sci);
1072
1073 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1074
1075 /* MACsec ethertype */
1076 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
1077 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_MACSEC);
1078
1079 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5;
1080
1081 /* Sectag AN + TCI SC present bit*/
1082 MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0,
1083 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1084 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0,
1085 tci_an << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1086
1087 if (sci_present) {
1088 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1089 misc_parameters_5.macsec_tag_2);
1090 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_2,
1091 be32_to_cpu(sci_p[0]));
1092
1093 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1094 misc_parameters_5.macsec_tag_3);
1095 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_3,
1096 be32_to_cpu(sci_p[1]));
1097 } else {
1098 /* When SCI isn't present in the Sectag, need to match the source */
1099 /* MAC address only if the SCI contains the default MACsec PORT */
1100 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_47_16);
1101 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_15_0);
1102 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.smac_47_16),
1103 sci_p, ETH_ALEN);
1104 }
1105
1106 crypto_params->type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC;
1107 crypto_params->obj_id = attrs->macsec_obj_id;
1108 }
1109
1110 static union mlx5e_macsec_rule *
macsec_fs_rx_add_rule(struct mlx5e_macsec_fs * macsec_fs,const struct macsec_context * macsec_ctx,struct mlx5_macsec_rule_attrs * attrs,u32 fs_id)1111 macsec_fs_rx_add_rule(struct mlx5e_macsec_fs *macsec_fs,
1112 const struct macsec_context *macsec_ctx,
1113 struct mlx5_macsec_rule_attrs *attrs,
1114 u32 fs_id)
1115 {
1116 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
1117 struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
1118 struct net_device *netdev = macsec_fs->netdev;
1119 union mlx5e_macsec_rule *macsec_rule = NULL;
1120 struct mlx5_modify_hdr *modify_hdr = NULL;
1121 struct mlx5_flow_destination dest = {};
1122 struct mlx5e_macsec_tables *rx_tables;
1123 struct mlx5e_macsec_rx_rule *rx_rule;
1124 struct mlx5_flow_act flow_act = {};
1125 struct mlx5e_flow_table *ft_crypto;
1126 struct mlx5_flow_handle *rule;
1127 struct mlx5_flow_spec *spec;
1128 int err = 0;
1129
1130 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1131 if (!spec)
1132 return NULL;
1133
1134 err = macsec_fs_rx_ft_get(macsec_fs);
1135 if (err)
1136 goto out_spec;
1137
1138 macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL);
1139 if (!macsec_rule) {
1140 macsec_fs_rx_ft_put(macsec_fs);
1141 goto out_spec;
1142 }
1143
1144 rx_rule = &macsec_rule->rx_rule;
1145 rx_tables = &rx_fs->tables;
1146 ft_crypto = &rx_tables->ft_crypto;
1147
1148 /* Set bit[31 - 30] macsec marker - 0x01 */
1149 /* Set bit[15-0] fs id */
1150 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
1151 MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
1152 MLX5_SET(set_action_in, action, data, MLX5_MACSEC_RX_METADAT_HANDLE(fs_id) | BIT(30));
1153 MLX5_SET(set_action_in, action, offset, 0);
1154 MLX5_SET(set_action_in, action, length, 32);
1155
1156 modify_hdr = mlx5_modify_header_alloc(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
1157 1, action);
1158 if (IS_ERR(modify_hdr)) {
1159 err = PTR_ERR(modify_hdr);
1160 netdev_err(netdev, "fail to alloc MACsec set modify_header_id err=%d\n", err);
1161 modify_hdr = NULL;
1162 goto err;
1163 }
1164 rx_rule->meta_modhdr = modify_hdr;
1165
1166 /* Rx crypto table with SCI rule */
1167 macsec_fs_rx_setup_fte(spec, &flow_act, attrs, true);
1168
1169 flow_act.modify_hdr = modify_hdr;
1170 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1171 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
1172 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1173
1174 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1175 dest.ft = rx_tables->ft_check;
1176 rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1);
1177 if (IS_ERR(rule)) {
1178 err = PTR_ERR(rule);
1179 netdev_err(netdev,
1180 "Failed to add SA with SCI rule to Rx crypto rule, err=%d\n",
1181 err);
1182 goto err;
1183 }
1184 rx_rule->rule[0] = rule;
1185
1186 /* Rx crypto table without SCI rule */
1187 if ((cpu_to_be64((__force u64)attrs->sci) & 0xFFFF) == ntohs(MACSEC_PORT_ES)) {
1188 memset(spec, 0, sizeof(struct mlx5_flow_spec));
1189 memset(&dest, 0, sizeof(struct mlx5_flow_destination));
1190 memset(&flow_act, 0, sizeof(flow_act));
1191
1192 macsec_fs_rx_setup_fte(spec, &flow_act, attrs, false);
1193
1194 flow_act.modify_hdr = modify_hdr;
1195 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1196 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
1197 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1198
1199 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1200 dest.ft = rx_tables->ft_check;
1201 rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1);
1202 if (IS_ERR(rule)) {
1203 err = PTR_ERR(rule);
1204 netdev_err(netdev,
1205 "Failed to add SA without SCI rule to Rx crypto rule, err=%d\n",
1206 err);
1207 goto err;
1208 }
1209 rx_rule->rule[1] = rule;
1210 }
1211
1212 kvfree(spec);
1213 return macsec_rule;
1214
1215 err:
1216 macsec_fs_rx_del_rule(macsec_fs, rx_rule);
1217 macsec_rule = NULL;
1218 out_spec:
1219 kvfree(spec);
1220 return macsec_rule;
1221 }
1222
macsec_fs_rx_init(struct mlx5e_macsec_fs * macsec_fs)1223 static int macsec_fs_rx_init(struct mlx5e_macsec_fs *macsec_fs)
1224 {
1225 struct net_device *netdev = macsec_fs->netdev;
1226 struct mlx5_core_dev *mdev = macsec_fs->mdev;
1227 struct mlx5e_macsec_tables *rx_tables;
1228 struct mlx5e_macsec_rx *rx_fs;
1229 struct mlx5_fc *flow_counter;
1230 int err;
1231
1232 rx_fs = kzalloc(sizeof(*rx_fs), GFP_KERNEL);
1233 if (!rx_fs)
1234 return -ENOMEM;
1235
1236 flow_counter = mlx5_fc_create(mdev, false);
1237 if (IS_ERR(flow_counter)) {
1238 err = PTR_ERR(flow_counter);
1239 netdev_err(netdev,
1240 "Failed to create MACsec Rx encrypt flow counter, err(%d)\n",
1241 err);
1242 goto err_encrypt_counter;
1243 }
1244
1245 rx_tables = &rx_fs->tables;
1246 rx_tables->check_rule_counter = flow_counter;
1247
1248 flow_counter = mlx5_fc_create(mdev, false);
1249 if (IS_ERR(flow_counter)) {
1250 err = PTR_ERR(flow_counter);
1251 netdev_err(netdev,
1252 "Failed to create MACsec Rx drop flow counter, err(%d)\n",
1253 err);
1254 goto err_drop_counter;
1255 }
1256 rx_tables->check_miss_rule_counter = flow_counter;
1257
1258 macsec_fs->rx_fs = rx_fs;
1259
1260 return 0;
1261
1262 err_drop_counter:
1263 mlx5_fc_destroy(mdev, rx_tables->check_rule_counter);
1264 rx_tables->check_rule_counter = NULL;
1265
1266 err_encrypt_counter:
1267 kfree(rx_fs);
1268 macsec_fs->rx_fs = NULL;
1269
1270 return err;
1271 }
1272
macsec_fs_rx_cleanup(struct mlx5e_macsec_fs * macsec_fs)1273 static void macsec_fs_rx_cleanup(struct mlx5e_macsec_fs *macsec_fs)
1274 {
1275 struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
1276 struct mlx5_core_dev *mdev = macsec_fs->mdev;
1277 struct mlx5e_macsec_tables *rx_tables;
1278
1279 if (!rx_fs)
1280 return;
1281
1282 rx_tables = &rx_fs->tables;
1283
1284 if (rx_tables->refcnt) {
1285 netdev_err(macsec_fs->netdev,
1286 "Can't destroy MACsec offload rx_fs, refcnt(%u) isn't 0\n",
1287 rx_tables->refcnt);
1288 return;
1289 }
1290
1291 if (rx_tables->check_miss_rule_counter) {
1292 mlx5_fc_destroy(mdev, rx_tables->check_miss_rule_counter);
1293 rx_tables->check_miss_rule_counter = NULL;
1294 }
1295
1296 if (rx_tables->check_rule_counter) {
1297 mlx5_fc_destroy(mdev, rx_tables->check_rule_counter);
1298 rx_tables->check_rule_counter = NULL;
1299 }
1300
1301 kfree(rx_fs);
1302 macsec_fs->rx_fs = NULL;
1303 }
1304
mlx5e_macsec_fs_get_stats_fill(struct mlx5e_macsec_fs * macsec_fs,void * macsec_stats)1305 void mlx5e_macsec_fs_get_stats_fill(struct mlx5e_macsec_fs *macsec_fs, void *macsec_stats)
1306 {
1307 struct mlx5e_macsec_stats *stats = (struct mlx5e_macsec_stats *)macsec_stats;
1308 struct mlx5e_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables;
1309 struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
1310 struct mlx5_core_dev *mdev = macsec_fs->mdev;
1311
1312 if (tx_tables->check_rule_counter)
1313 mlx5_fc_query(mdev, tx_tables->check_rule_counter,
1314 &stats->macsec_tx_pkts, &stats->macsec_tx_bytes);
1315
1316 if (tx_tables->check_miss_rule_counter)
1317 mlx5_fc_query(mdev, tx_tables->check_miss_rule_counter,
1318 &stats->macsec_tx_pkts_drop, &stats->macsec_tx_bytes_drop);
1319
1320 if (rx_tables->check_rule_counter)
1321 mlx5_fc_query(mdev, rx_tables->check_rule_counter,
1322 &stats->macsec_rx_pkts, &stats->macsec_rx_bytes);
1323
1324 if (rx_tables->check_miss_rule_counter)
1325 mlx5_fc_query(mdev, rx_tables->check_miss_rule_counter,
1326 &stats->macsec_rx_pkts_drop, &stats->macsec_rx_bytes_drop);
1327 }
1328
1329 union mlx5e_macsec_rule *
mlx5e_macsec_fs_add_rule(struct mlx5e_macsec_fs * macsec_fs,const struct macsec_context * macsec_ctx,struct mlx5_macsec_rule_attrs * attrs,u32 * sa_fs_id)1330 mlx5e_macsec_fs_add_rule(struct mlx5e_macsec_fs *macsec_fs,
1331 const struct macsec_context *macsec_ctx,
1332 struct mlx5_macsec_rule_attrs *attrs,
1333 u32 *sa_fs_id)
1334 {
1335 return (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ?
1336 macsec_fs_tx_add_rule(macsec_fs, macsec_ctx, attrs, sa_fs_id) :
1337 macsec_fs_rx_add_rule(macsec_fs, macsec_ctx, attrs, *sa_fs_id);
1338 }
1339
mlx5e_macsec_fs_del_rule(struct mlx5e_macsec_fs * macsec_fs,union mlx5e_macsec_rule * macsec_rule,int action)1340 void mlx5e_macsec_fs_del_rule(struct mlx5e_macsec_fs *macsec_fs,
1341 union mlx5e_macsec_rule *macsec_rule,
1342 int action)
1343 {
1344 (action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ?
1345 macsec_fs_tx_del_rule(macsec_fs, &macsec_rule->tx_rule) :
1346 macsec_fs_rx_del_rule(macsec_fs, &macsec_rule->rx_rule);
1347 }
1348
mlx5e_macsec_fs_cleanup(struct mlx5e_macsec_fs * macsec_fs)1349 void mlx5e_macsec_fs_cleanup(struct mlx5e_macsec_fs *macsec_fs)
1350 {
1351 macsec_fs_rx_cleanup(macsec_fs);
1352 macsec_fs_tx_cleanup(macsec_fs);
1353 kfree(macsec_fs);
1354 }
1355
1356 struct mlx5e_macsec_fs *
mlx5e_macsec_fs_init(struct mlx5_core_dev * mdev,struct net_device * netdev)1357 mlx5e_macsec_fs_init(struct mlx5_core_dev *mdev,
1358 struct net_device *netdev)
1359 {
1360 struct mlx5e_macsec_fs *macsec_fs;
1361 int err;
1362
1363 macsec_fs = kzalloc(sizeof(*macsec_fs), GFP_KERNEL);
1364 if (!macsec_fs)
1365 return NULL;
1366
1367 macsec_fs->mdev = mdev;
1368 macsec_fs->netdev = netdev;
1369
1370 err = macsec_fs_tx_init(macsec_fs);
1371 if (err) {
1372 netdev_err(netdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err);
1373 goto err;
1374 }
1375
1376 err = macsec_fs_rx_init(macsec_fs);
1377 if (err) {
1378 netdev_err(netdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err);
1379 goto tx_cleanup;
1380 }
1381
1382 return macsec_fs;
1383
1384 tx_cleanup:
1385 macsec_fs_tx_cleanup(macsec_fs);
1386 err:
1387 kfree(macsec_fs);
1388 return NULL;
1389 }
1390