1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Atlantic Network Driver
3  *
4  * Copyright (C) 2014-2019 aQuantia Corporation
5  * Copyright (C) 2019-2020 Marvell International Ltd.
6  */
7 
8 /* File aq_main.c: Main file for aQuantia Linux driver. */
9 
10 #include "aq_main.h"
11 #include "aq_nic.h"
12 #include "aq_pci_func.h"
13 #include "aq_ethtool.h"
14 #include "aq_ptp.h"
15 #include "aq_filters.h"
16 #include "aq_hw_utils.h"
17 #include "aq_vec.h"
18 
19 #include <linux/netdevice.h>
20 #include <linux/module.h>
21 #include <linux/ip.h>
22 #include <linux/udp.h>
23 #include <net/pkt_cls.h>
24 #include <net/pkt_sched.h>
25 #include <linux/filter.h>
26 
27 MODULE_LICENSE("GPL v2");
28 MODULE_AUTHOR(AQ_CFG_DRV_AUTHOR);
29 MODULE_DESCRIPTION(AQ_CFG_DRV_DESC);
30 
31 DEFINE_STATIC_KEY_FALSE(aq_xdp_locking_key);
32 EXPORT_SYMBOL(aq_xdp_locking_key);
33 
34 static const char aq_ndev_driver_name[] = AQ_CFG_DRV_NAME;
35 
36 static const struct net_device_ops aq_ndev_ops;
37 
38 static struct workqueue_struct *aq_ndev_wq;
39 
aq_ndev_schedule_work(struct work_struct * work)40 void aq_ndev_schedule_work(struct work_struct *work)
41 {
42 	queue_work(aq_ndev_wq, work);
43 }
44 
aq_ndev_alloc(void)45 struct net_device *aq_ndev_alloc(void)
46 {
47 	struct net_device *ndev = NULL;
48 	struct aq_nic_s *aq_nic = NULL;
49 
50 	ndev = alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_HW_QUEUES_MAX);
51 	if (!ndev)
52 		return NULL;
53 
54 	aq_nic = netdev_priv(ndev);
55 	aq_nic->ndev = ndev;
56 	ndev->netdev_ops = &aq_ndev_ops;
57 	ndev->ethtool_ops = &aq_ethtool_ops;
58 
59 	return ndev;
60 }
61 
aq_ndev_open(struct net_device * ndev)62 int aq_ndev_open(struct net_device *ndev)
63 {
64 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
65 	int err = 0;
66 
67 	err = aq_nic_init(aq_nic);
68 	if (err < 0)
69 		goto err_exit;
70 
71 	err = aq_reapply_rxnfc_all_rules(aq_nic);
72 	if (err < 0)
73 		goto err_exit;
74 
75 	err = aq_filters_vlans_update(aq_nic);
76 	if (err < 0)
77 		goto err_exit;
78 
79 	err = aq_nic_start(aq_nic);
80 	if (err < 0) {
81 		aq_nic_stop(aq_nic);
82 		goto err_exit;
83 	}
84 
85 err_exit:
86 	if (err < 0)
87 		aq_nic_deinit(aq_nic, true);
88 
89 	return err;
90 }
91 
aq_ndev_close(struct net_device * ndev)92 int aq_ndev_close(struct net_device *ndev)
93 {
94 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
95 	int err = 0;
96 
97 	err = aq_nic_stop(aq_nic);
98 	aq_nic_deinit(aq_nic, true);
99 
100 	return err;
101 }
102 
aq_ndev_start_xmit(struct sk_buff * skb,struct net_device * ndev)103 static netdev_tx_t aq_ndev_start_xmit(struct sk_buff *skb, struct net_device *ndev)
104 {
105 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
106 
107 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
108 	if (unlikely(aq_utils_obj_test(&aq_nic->flags, AQ_NIC_PTP_DPATH_UP))) {
109 		/* Hardware adds the Timestamp for PTPv2 802.AS1
110 		 * and PTPv2 IPv4 UDP.
111 		 * We have to push even general 320 port messages to the ptp
112 		 * queue explicitly. This is a limitation of current firmware
113 		 * and hardware PTP design of the chip. Otherwise ptp stream
114 		 * will fail to sync
115 		 */
116 		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ||
117 		    unlikely((ip_hdr(skb)->version == 4) &&
118 			     (ip_hdr(skb)->protocol == IPPROTO_UDP) &&
119 			     ((udp_hdr(skb)->dest == htons(319)) ||
120 			      (udp_hdr(skb)->dest == htons(320)))) ||
121 		    unlikely(eth_hdr(skb)->h_proto == htons(ETH_P_1588)))
122 			return aq_ptp_xmit(aq_nic, skb);
123 	}
124 #endif
125 
126 	skb_tx_timestamp(skb);
127 	return aq_nic_xmit(aq_nic, skb);
128 }
129 
aq_ndev_change_mtu(struct net_device * ndev,int new_mtu)130 static int aq_ndev_change_mtu(struct net_device *ndev, int new_mtu)
131 {
132 	int new_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;
133 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
134 	struct bpf_prog *prog;
135 	int err;
136 
137 	prog = READ_ONCE(aq_nic->xdp_prog);
138 	if (prog && !prog->aux->xdp_has_frags &&
139 	    new_frame_size > AQ_CFG_RX_FRAME_MAX) {
140 		netdev_err(ndev, "Illegal MTU %d for XDP prog without frags\n",
141 			   ndev->mtu);
142 		return -EOPNOTSUPP;
143 	}
144 
145 	err = aq_nic_set_mtu(aq_nic, new_mtu + ETH_HLEN);
146 
147 	if (err < 0)
148 		goto err_exit;
149 	ndev->mtu = new_mtu;
150 
151 err_exit:
152 	return err;
153 }
154 
aq_ndev_set_features(struct net_device * ndev,netdev_features_t features)155 static int aq_ndev_set_features(struct net_device *ndev,
156 				netdev_features_t features)
157 {
158 	bool is_vlan_tx_insert = !!(features & NETIF_F_HW_VLAN_CTAG_TX);
159 	bool is_vlan_rx_strip = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
160 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
161 	bool need_ndev_restart = false;
162 	struct aq_nic_cfg_s *aq_cfg;
163 	bool is_lro = false;
164 	int err = 0;
165 
166 	aq_cfg = aq_nic_get_cfg(aq_nic);
167 
168 	if (!(features & NETIF_F_NTUPLE)) {
169 		if (aq_nic->ndev->features & NETIF_F_NTUPLE) {
170 			err = aq_clear_rxnfc_all_rules(aq_nic);
171 			if (unlikely(err))
172 				goto err_exit;
173 		}
174 	}
175 	if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER)) {
176 		if (aq_nic->ndev->features & NETIF_F_HW_VLAN_CTAG_FILTER) {
177 			err = aq_filters_vlan_offload_off(aq_nic);
178 			if (unlikely(err))
179 				goto err_exit;
180 		}
181 	}
182 
183 	aq_cfg->features = features;
184 
185 	if (aq_cfg->aq_hw_caps->hw_features & NETIF_F_LRO) {
186 		is_lro = features & NETIF_F_LRO;
187 
188 		if (aq_cfg->is_lro != is_lro) {
189 			aq_cfg->is_lro = is_lro;
190 			need_ndev_restart = true;
191 		}
192 	}
193 
194 	if ((aq_nic->ndev->features ^ features) & NETIF_F_RXCSUM) {
195 		err = aq_nic->aq_hw_ops->hw_set_offload(aq_nic->aq_hw,
196 							aq_cfg);
197 
198 		if (unlikely(err))
199 			goto err_exit;
200 	}
201 
202 	if (aq_cfg->is_vlan_rx_strip != is_vlan_rx_strip) {
203 		aq_cfg->is_vlan_rx_strip = is_vlan_rx_strip;
204 		need_ndev_restart = true;
205 	}
206 	if (aq_cfg->is_vlan_tx_insert != is_vlan_tx_insert) {
207 		aq_cfg->is_vlan_tx_insert = is_vlan_tx_insert;
208 		need_ndev_restart = true;
209 	}
210 
211 	if (need_ndev_restart && netif_running(ndev)) {
212 		aq_ndev_close(ndev);
213 		aq_ndev_open(ndev);
214 	}
215 
216 err_exit:
217 	return err;
218 }
219 
aq_ndev_fix_features(struct net_device * ndev,netdev_features_t features)220 static netdev_features_t aq_ndev_fix_features(struct net_device *ndev,
221 					      netdev_features_t features)
222 {
223 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
224 	struct bpf_prog *prog;
225 
226 	if (!(features & NETIF_F_RXCSUM))
227 		features &= ~NETIF_F_LRO;
228 
229 	prog = READ_ONCE(aq_nic->xdp_prog);
230 	if (prog && !prog->aux->xdp_has_frags &&
231 	    aq_nic->xdp_prog && features & NETIF_F_LRO) {
232 		netdev_err(ndev, "LRO is not supported with single buffer XDP, disabling\n");
233 		features &= ~NETIF_F_LRO;
234 	}
235 
236 	return features;
237 }
238 
aq_ndev_set_mac_address(struct net_device * ndev,void * addr)239 static int aq_ndev_set_mac_address(struct net_device *ndev, void *addr)
240 {
241 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
242 	int err = 0;
243 
244 	err = eth_mac_addr(ndev, addr);
245 	if (err < 0)
246 		goto err_exit;
247 	err = aq_nic_set_mac(aq_nic, ndev);
248 	if (err < 0)
249 		goto err_exit;
250 
251 err_exit:
252 	return err;
253 }
254 
aq_ndev_set_multicast_settings(struct net_device * ndev)255 static void aq_ndev_set_multicast_settings(struct net_device *ndev)
256 {
257 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
258 
259 	(void)aq_nic_set_multicast_list(aq_nic, ndev);
260 }
261 
262 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
aq_ndev_config_hwtstamp(struct aq_nic_s * aq_nic,struct hwtstamp_config * config)263 static int aq_ndev_config_hwtstamp(struct aq_nic_s *aq_nic,
264 				   struct hwtstamp_config *config)
265 {
266 	switch (config->tx_type) {
267 	case HWTSTAMP_TX_OFF:
268 	case HWTSTAMP_TX_ON:
269 		break;
270 	default:
271 		return -ERANGE;
272 	}
273 
274 	switch (config->rx_filter) {
275 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
276 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
277 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
278 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
279 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
280 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
281 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
282 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
283 		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
284 		break;
285 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
286 	case HWTSTAMP_FILTER_NONE:
287 		break;
288 	default:
289 		return -ERANGE;
290 	}
291 
292 	return aq_ptp_hwtstamp_config_set(aq_nic->aq_ptp, config);
293 }
294 #endif
295 
aq_ndev_hwtstamp_set(struct aq_nic_s * aq_nic,struct ifreq * ifr)296 static int aq_ndev_hwtstamp_set(struct aq_nic_s *aq_nic, struct ifreq *ifr)
297 {
298 	struct hwtstamp_config config;
299 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
300 	int ret_val;
301 #endif
302 
303 	if (!aq_nic->aq_ptp)
304 		return -EOPNOTSUPP;
305 
306 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
307 		return -EFAULT;
308 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
309 	ret_val = aq_ndev_config_hwtstamp(aq_nic, &config);
310 	if (ret_val)
311 		return ret_val;
312 #endif
313 
314 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
315 	       -EFAULT : 0;
316 }
317 
318 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
aq_ndev_hwtstamp_get(struct aq_nic_s * aq_nic,struct ifreq * ifr)319 static int aq_ndev_hwtstamp_get(struct aq_nic_s *aq_nic, struct ifreq *ifr)
320 {
321 	struct hwtstamp_config config;
322 
323 	if (!aq_nic->aq_ptp)
324 		return -EOPNOTSUPP;
325 
326 	aq_ptp_hwtstamp_config_get(aq_nic->aq_ptp, &config);
327 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
328 	       -EFAULT : 0;
329 }
330 #endif
331 
aq_ndev_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)332 static int aq_ndev_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
333 {
334 	struct aq_nic_s *aq_nic = netdev_priv(netdev);
335 
336 	switch (cmd) {
337 	case SIOCSHWTSTAMP:
338 		return aq_ndev_hwtstamp_set(aq_nic, ifr);
339 
340 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
341 	case SIOCGHWTSTAMP:
342 		return aq_ndev_hwtstamp_get(aq_nic, ifr);
343 #endif
344 	}
345 
346 	return -EOPNOTSUPP;
347 }
348 
aq_ndo_vlan_rx_add_vid(struct net_device * ndev,__be16 proto,u16 vid)349 static int aq_ndo_vlan_rx_add_vid(struct net_device *ndev, __be16 proto,
350 				  u16 vid)
351 {
352 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
353 
354 	if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
355 		return -EOPNOTSUPP;
356 
357 	set_bit(vid, aq_nic->active_vlans);
358 
359 	return aq_filters_vlans_update(aq_nic);
360 }
361 
aq_ndo_vlan_rx_kill_vid(struct net_device * ndev,__be16 proto,u16 vid)362 static int aq_ndo_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto,
363 				   u16 vid)
364 {
365 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
366 
367 	if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
368 		return -EOPNOTSUPP;
369 
370 	clear_bit(vid, aq_nic->active_vlans);
371 
372 	if (-ENOENT == aq_del_fvlan_by_vlan(aq_nic, vid))
373 		return aq_filters_vlans_update(aq_nic);
374 
375 	return 0;
376 }
377 
aq_validate_mqprio_opt(struct aq_nic_s * self,struct tc_mqprio_qopt_offload * mqprio,const unsigned int num_tc)378 static int aq_validate_mqprio_opt(struct aq_nic_s *self,
379 				  struct tc_mqprio_qopt_offload *mqprio,
380 				  const unsigned int num_tc)
381 {
382 	const bool has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
383 	struct aq_nic_cfg_s *aq_nic_cfg = aq_nic_get_cfg(self);
384 	const unsigned int tcs_max = min_t(u8, aq_nic_cfg->aq_hw_caps->tcs_max,
385 					   AQ_CFG_TCS_MAX);
386 
387 	if (num_tc > tcs_max) {
388 		netdev_err(self->ndev, "Too many TCs requested\n");
389 		return -EOPNOTSUPP;
390 	}
391 
392 	if (num_tc != 0 && !is_power_of_2(num_tc)) {
393 		netdev_err(self->ndev, "TC count should be power of 2\n");
394 		return -EOPNOTSUPP;
395 	}
396 
397 	if (has_min_rate && !ATL_HW_IS_CHIP_FEATURE(self->aq_hw, ANTIGUA)) {
398 		netdev_err(self->ndev, "Min tx rate is not supported\n");
399 		return -EOPNOTSUPP;
400 	}
401 
402 	return 0;
403 }
404 
aq_ndo_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)405 static int aq_ndo_setup_tc(struct net_device *dev, enum tc_setup_type type,
406 			   void *type_data)
407 {
408 	struct tc_mqprio_qopt_offload *mqprio = type_data;
409 	struct aq_nic_s *aq_nic = netdev_priv(dev);
410 	bool has_min_rate;
411 	bool has_max_rate;
412 	int err;
413 	int i;
414 
415 	if (type != TC_SETUP_QDISC_MQPRIO)
416 		return -EOPNOTSUPP;
417 
418 	has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
419 	has_max_rate = !!(mqprio->flags & TC_MQPRIO_F_MAX_RATE);
420 
421 	err = aq_validate_mqprio_opt(aq_nic, mqprio, mqprio->qopt.num_tc);
422 	if (err)
423 		return err;
424 
425 	for (i = 0; i < mqprio->qopt.num_tc; i++) {
426 		if (has_max_rate) {
427 			u64 max_rate = mqprio->max_rate[i];
428 
429 			do_div(max_rate, AQ_MBPS_DIVISOR);
430 			aq_nic_setup_tc_max_rate(aq_nic, i, (u32)max_rate);
431 		}
432 
433 		if (has_min_rate) {
434 			u64 min_rate = mqprio->min_rate[i];
435 
436 			do_div(min_rate, AQ_MBPS_DIVISOR);
437 			aq_nic_setup_tc_min_rate(aq_nic, i, (u32)min_rate);
438 		}
439 	}
440 
441 	return aq_nic_setup_tc_mqprio(aq_nic, mqprio->qopt.num_tc,
442 				      mqprio->qopt.prio_tc_map);
443 }
444 
aq_xdp_setup(struct net_device * ndev,struct bpf_prog * prog,struct netlink_ext_ack * extack)445 static int aq_xdp_setup(struct net_device *ndev, struct bpf_prog *prog,
446 			struct netlink_ext_ack *extack)
447 {
448 	bool need_update, running = netif_running(ndev);
449 	struct aq_nic_s *aq_nic = netdev_priv(ndev);
450 	struct bpf_prog *old_prog;
451 
452 	if (prog && !prog->aux->xdp_has_frags) {
453 		if (ndev->mtu > AQ_CFG_RX_FRAME_MAX) {
454 			NL_SET_ERR_MSG_MOD(extack,
455 					   "prog does not support XDP frags");
456 			return -EOPNOTSUPP;
457 		}
458 
459 		if (prog && ndev->features & NETIF_F_LRO) {
460 			netdev_err(ndev,
461 				   "LRO is not supported with single buffer XDP, disabling\n");
462 			ndev->features &= ~NETIF_F_LRO;
463 		}
464 	}
465 
466 	need_update = !!aq_nic->xdp_prog != !!prog;
467 	if (running && need_update)
468 		aq_ndev_close(ndev);
469 
470 	old_prog = xchg(&aq_nic->xdp_prog, prog);
471 	if (old_prog)
472 		bpf_prog_put(old_prog);
473 
474 	if (!old_prog && prog)
475 		static_branch_inc(&aq_xdp_locking_key);
476 	else if (old_prog && !prog)
477 		static_branch_dec(&aq_xdp_locking_key);
478 
479 	if (running && need_update)
480 		return aq_ndev_open(ndev);
481 
482 	return 0;
483 }
484 
aq_xdp(struct net_device * dev,struct netdev_bpf * xdp)485 static int aq_xdp(struct net_device *dev, struct netdev_bpf *xdp)
486 {
487 	switch (xdp->command) {
488 	case XDP_SETUP_PROG:
489 		return aq_xdp_setup(dev, xdp->prog, xdp->extack);
490 	default:
491 		return -EINVAL;
492 	}
493 }
494 
495 static const struct net_device_ops aq_ndev_ops = {
496 	.ndo_open = aq_ndev_open,
497 	.ndo_stop = aq_ndev_close,
498 	.ndo_start_xmit = aq_ndev_start_xmit,
499 	.ndo_set_rx_mode = aq_ndev_set_multicast_settings,
500 	.ndo_change_mtu = aq_ndev_change_mtu,
501 	.ndo_set_mac_address = aq_ndev_set_mac_address,
502 	.ndo_set_features = aq_ndev_set_features,
503 	.ndo_fix_features = aq_ndev_fix_features,
504 	.ndo_eth_ioctl = aq_ndev_ioctl,
505 	.ndo_vlan_rx_add_vid = aq_ndo_vlan_rx_add_vid,
506 	.ndo_vlan_rx_kill_vid = aq_ndo_vlan_rx_kill_vid,
507 	.ndo_setup_tc = aq_ndo_setup_tc,
508 	.ndo_bpf = aq_xdp,
509 	.ndo_xdp_xmit = aq_xdp_xmit,
510 };
511 
aq_ndev_init_module(void)512 static int __init aq_ndev_init_module(void)
513 {
514 	int ret;
515 
516 	aq_ndev_wq = create_singlethread_workqueue(aq_ndev_driver_name);
517 	if (!aq_ndev_wq) {
518 		pr_err("Failed to create workqueue\n");
519 		return -ENOMEM;
520 	}
521 
522 	ret = aq_pci_func_register_driver();
523 	if (ret) {
524 		destroy_workqueue(aq_ndev_wq);
525 		return ret;
526 	}
527 
528 	return 0;
529 }
530 
aq_ndev_exit_module(void)531 static void __exit aq_ndev_exit_module(void)
532 {
533 	aq_pci_func_unregister_driver();
534 
535 	if (aq_ndev_wq) {
536 		destroy_workqueue(aq_ndev_wq);
537 		aq_ndev_wq = NULL;
538 	}
539 }
540 
541 module_init(aq_ndev_init_module);
542 module_exit(aq_ndev_exit_module);
543