35 #include <linux/tcp.h>
36 #include <linux/if_vlan.h>
38 #include <linux/slab.h>
39 #include <linux/hash.h>
59 netdev_set_num_tc(dev, up);
63 for (i = 0; i <
up; i++) {
64 netdev_set_tc_queue(dev, i, q, offset);
71 #ifdef CONFIG_RFS_ACCEL
73 struct mlx4_en_filter {
93 static void mlx4_en_filter_rfs_expire(
struct mlx4_en_priv *priv);
98 struct mlx4_en_filter,
105 .dst_port = filter->dst_port,
107 .src_port = filter->src_port,
116 .dst_ip = filter->dst_ip,
118 .src_ip = filter->src_ip,
145 rule.
qpn = priv->
rss_map.qps[filter->rxq_index].qpn;
149 filter->activated = 0;
151 if (filter->reg_id) {
154 en_err(priv,
"Error detaching flow. rc = %d\n", rc);
159 en_err(priv,
"Error attaching flow. err = %d\n", rc);
161 mlx4_en_filter_rfs_expire(priv);
163 filter->activated = 1;
174 ((
__force unsigned long)dst_port << 2);
179 return &priv->filter_hash[bucket_idx];
182 static struct mlx4_en_filter *
187 struct mlx4_en_filter *filter =
NULL;
189 filter = kzalloc(
sizeof(
struct mlx4_en_filter),
GFP_ATOMIC);
194 filter->rxq_index = rxq_index;
195 INIT_WORK(&filter->work, mlx4_en_filter_work);
197 filter->src_ip = src_ip;
198 filter->dst_ip = dst_ip;
199 filter->src_port = src_port;
200 filter->dst_port = dst_port;
202 filter->flow_id = flow_id;
204 filter->id = priv->last_filter_id++ % RPS_NO_FILTER;
207 hlist_add_head(&filter->filter_chain,
208 filter_hash_bucket(priv, src_ip, dst_ip, src_port,
214 static void mlx4_en_filter_free(
struct mlx4_en_filter *filter)
223 en_err(priv,
"Error detaching flow. rc = %d\n", rc);
228 static inline struct mlx4_en_filter *
233 struct mlx4_en_filter *
filter;
234 struct mlx4_en_filter *
ret =
NULL;
237 filter_hash_bucket(priv, src_ip, dst_ip,
240 if (filter->src_ip == src_ip &&
241 filter->dst_ip == dst_ip &&
242 filter->src_port == src_port &&
243 filter->dst_port == dst_port) {
254 u16 rxq_index,
u32 flow_id)
257 struct mlx4_en_filter *
filter;
264 int nhoff = skb_network_offset(skb);
270 ip = (
const struct iphdr *)(skb->
data + nhoff);
271 if (ip_is_fragment(ip))
274 ports = (
const __be16 *)(skb->
data + nhoff + 4 * ip->ihl);
284 spin_lock_bh(&priv->filters_lock);
285 filter = mlx4_en_filter_find(priv, src_ip, dst_ip, src_port, dst_port);
287 if (filter->rxq_index == rxq_index)
290 filter->rxq_index = rxq_index;
292 filter = mlx4_en_filter_alloc(priv, rxq_index,
294 src_port, dst_port, flow_id);
306 spin_unlock_bh(&priv->filters_lock);
317 spin_lock_bh(&priv->filters_lock);
319 list_move(&filter->next, &del_list);
320 hlist_del(&filter->filter_chain);
322 spin_unlock_bh(&priv->filters_lock);
326 mlx4_en_filter_free(filter);
330 static void mlx4_en_filter_rfs_expire(
struct mlx4_en_priv *priv)
332 struct mlx4_en_filter *filter =
NULL, *
tmp, *last_filter =
NULL;
336 spin_lock_bh(&priv->filters_lock);
341 if (filter->activated &&
343 rps_may_expire_flow(priv->
dev,
344 filter->rxq_index, filter->flow_id,
346 list_move(&filter->next, &del_list);
347 hlist_del(&filter->filter_chain);
354 if (last_filter && (&last_filter->next != priv->filters.next))
355 list_move(&priv->filters, &last_filter->next);
357 spin_unlock_bh(&priv->filters_lock);
360 mlx4_en_filter_free(filter);
364 static int mlx4_en_vlan_rx_add_vid(
struct net_device *dev,
unsigned short vid)
371 en_dbg(HW, priv,
"adding VLAN:%d\n", vid);
380 en_err(priv,
"Failed configuring VLAN filter\n");
383 en_err(priv,
"failed adding vlan %d\n", vid);
389 static int mlx4_en_vlan_rx_kill_vid(
struct net_device *dev,
unsigned short vid)
396 en_dbg(HW, priv,
"Killing VID:%d\n", vid);
405 en_err(priv,
"could not find vid %d in cache\n", vid);
410 en_err(priv,
"Failed configuring VLAN filter\n");
435 if (!is_valid_ether_addr(saddr->
sa_data))
444 static void mlx4_en_do_set_mac(
struct work_struct *work)
457 en_err(priv,
"Failed changing HW MAC address\n");
459 en_dbg(HW, priv,
"Port is down while "
460 "registering mac, exiting...\n");
465 static void mlx4_en_clear_list(
struct net_device *dev)
476 static void mlx4_en_cache_mclist(
struct net_device *dev)
482 mlx4_en_clear_list(dev);
486 en_err(priv,
"failed to allocate multicast list\n");
487 mlx4_en_clear_list(dev);
495 static void update_mclist_flags(
struct mlx4_en_priv *priv,
533 en_err(priv,
"Failed to allocate current multicast list\n");
544 static void mlx4_en_set_multicast(
struct net_device *dev)
554 static void mlx4_en_do_set_multicast(
struct work_struct *work)
562 u8 mc_list[16] = {0};
567 en_dbg(HW, priv,
"Card is not up, "
568 "ignoring multicast change.\n");
572 en_dbg(HW, priv,
"Port is down, "
573 "ignoring multicast change.\n");
577 if (!netif_carrier_ok(dev)) {
594 en_warn(priv,
"Entering promiscuous mode\n");
598 switch (mdev->
dev->caps.steering_mode) {
605 en_err(priv,
"Failed enabling promiscuous mode\n");
614 en_err(priv,
"Failed enabling unicast promiscuous mode\n");
624 en_err(priv,
"Failed enabling multicast promiscuous mode\n");
635 en_err(priv,
"Failed enabling promiscuous mode\n");
643 en_err(priv,
"Failed disabling "
644 "multicast filter\n");
649 en_err(priv,
"Failed disabling VLAN filter\n");
660 en_warn(priv,
"Leaving promiscuous mode\n");
664 switch (mdev->
dev->caps.steering_mode) {
670 en_err(priv,
"Failed disabling promiscuous mode\n");
679 en_err(priv,
"Failed disabling unicast promiscuous mode\n");
686 en_err(priv,
"Failed disabling multicast promiscuous mode\n");
696 en_err(priv,
"Failed disabling promiscuous mode\n");
703 en_err(priv,
"Failed enabling VLAN filter\n");
711 en_err(priv,
"Failed disabling multicast filter\n");
715 switch (mdev->
dev->caps.steering_mode) {
733 en_err(priv,
"Failed entering multicast promisc mode\n");
739 switch (mdev->
dev->caps.steering_mode) {
756 en_err(priv,
"Failed disabling multicast promiscuous mode\n");
763 en_err(priv,
"Failed disabling multicast filter\n");
771 netif_tx_lock_bh(dev);
772 mlx4_en_cache_mclist(dev);
773 netif_tx_unlock_bh(dev);
782 en_err(priv,
"Failed enabling multicast filter\n");
789 mc_list[5] = priv->
port;
796 en_err(priv,
"Fail to detach multicast address\n");
805 mc_list[5] = priv->
port;
813 en_err(priv,
"Fail to attach multicast address\n");
822 #ifdef CONFIG_NET_POLL_CONTROLLER
823 static void mlx4_en_netpoll(
struct net_device *dev)
835 spin_unlock_irqrestore(&cq->
lock, flags);
840 static void mlx4_en_tx_timeout(
struct net_device *dev)
846 en_warn(priv,
"Tx timeout called on port:%d\n", priv->
port);
849 en_dbg(DRV, priv,
"Scheduling watchdog\n");
865 static void mlx4_en_set_default_moderation(
struct mlx4_en_priv *priv)
880 en_dbg(
INTR, priv,
"Default coalesing params for mtu:%d - "
881 "rx_frames:%d rx_usecs:%d\n",
911 static void mlx4_en_auto_moderation(
struct mlx4_en_priv *priv)
917 unsigned long avg_pkt_size;
920 unsigned long rx_pkt_diff;
933 rx_pkt_diff = ((
unsigned long) (rx_packets -
935 packets = rx_pkt_diff;
936 rate = packets * HZ /
period;
937 avg_pkt_size = packets ? ((
unsigned long) (rx_bytes -
944 if (rate < priv->pkt_rate_low)
963 en_err(priv,
"Failed modifying moderation "
964 "for cq:%d\n", ring);
973 static void mlx4_en_do_get_stats(
struct work_struct *work)
983 en_dbg(HW, priv,
"Could not update stats\n");
988 mlx4_en_auto_moderation(priv);
999 static void mlx4_en_linkstate(
struct work_struct *work)
1034 u8 mc_list[16] = {0};
1037 en_dbg(DRV, priv,
"start port called while port already up\n");
1041 INIT_LIST_HEAD(&priv->
mc_list);
1052 en_err(priv,
"Failed to activate RX rings\n");
1060 en_err(priv,
"Failed activating Rx CQ\n");
1063 for (j = 0; j < cq->
size; j++)
1067 en_err(priv,
"Failed setting cq moderation parameters");
1077 en_dbg(DRV, priv,
"Getting qp number for port %d\n", priv->
port);
1081 en_err(priv,
"Failed getting eth qp\n");
1088 en_err(priv,
"Failed configuring rss steering\n");
1102 en_err(priv,
"Failed allocating Tx CQ\n");
1107 en_err(priv,
"Failed setting cq moderation parameters");
1111 en_dbg(DRV, priv,
"Resetting index of collapsed CQ:%d to -1\n", i);
1117 i / priv->
mdev->profile.num_tx_rings_p_up);
1119 en_err(priv,
"Failed allocating Tx ring\n");
1123 tx_ring->
tx_queue = netdev_get_tx_queue(dev, i);
1130 *((
u32 *) (tx_ring->
buf + j)) = 0xffffffff;
1137 priv->
prof->tx_pause,
1139 priv->
prof->rx_pause,
1140 priv->
prof->rx_ppp);
1142 en_err(priv,
"Failed setting port general configurations "
1143 "for port %d, with error %d\n", priv->
port, err);
1149 en_err(priv,
"Failed setting default qp numbers\n");
1154 en_dbg(HW, priv,
"Initializing port\n");
1157 en_err(priv,
"Failed Initializing port\n");
1163 mc_list[5] = priv->
port;
1167 mlx4_warn(mdev,
"Failed Attaching Broadcast\n");
1171 if (mdev->
dev->caps.steering_mode ==
1187 netif_tx_start_all_queues(dev);
1191 while (tx_index--) {
1216 u8 mc_list[16] = {0};
1219 en_dbg(DRV, priv,
"stop port called while port already down\n");
1224 netif_tx_lock_bh(dev);
1225 netif_tx_stop_all_queues(dev);
1226 netif_tx_unlock_bh(dev);
1233 mc_list[5] = priv->
port;
1238 mc_list[5] = priv->
port;
1242 mlx4_en_clear_list(dev);
1282 static void mlx4_en_restart(
struct work_struct *work)
1290 en_dbg(DRV, priv,
"Watchdog task called for port %d\n", priv->
port);
1296 netdev_tx_reset_queue(priv->
tx_ring[i].tx_queue);
1298 en_err(priv,
"Failed restarting port %d\n", priv->
port);
1303 static void mlx4_en_clear_stats(
struct net_device *dev)
1310 en_dbg(HW, priv,
"Failed dumping statistics\n");
1330 static int mlx4_en_open(
struct net_device *dev)
1339 en_err(priv,
"Cannot open - device down/disabled\n");
1345 mlx4_en_clear_stats(dev);
1349 en_err(priv,
"Failed starting port:%d\n", priv->
port);
1357 static int mlx4_en_close(
struct net_device *dev)
1362 en_dbg(IFDOWN, priv,
"Close port called\n");
1377 #ifdef CONFIG_RFS_ACCEL
1378 free_irq_cpu_rmap(priv->
dev->rx_cpu_rmap);
1379 priv->
dev->rx_cpu_rmap =
NULL;
1385 if (priv->
tx_cq[i].buf)
1393 if (priv->
rx_cq[i].buf)
1411 en_err(priv,
"failed reserving range for TX rings\n");
1437 #ifdef CONFIG_RFS_ACCEL
1439 if (!priv->
dev->rx_cpu_rmap)
1442 INIT_LIST_HEAD(&priv->filters);
1449 en_err(priv,
"Failed to allocate NIC resources\n");
1459 en_dbg(DRV, priv,
"Destroying netdev on port:%d\n", priv->
port);
1485 static int mlx4_en_change_mtu(
struct net_device *dev,
int new_mtu)
1491 en_dbg(DRV, priv,
"Change MTU called - current:%d new:%d\n",
1495 en_err(priv,
"Bad MTU size:%d.\n", new_mtu);
1500 if (netif_running(dev)) {
1505 en_dbg(DRV, priv,
"Change MTU called with card down!?\n");
1510 en_err(priv,
"Failed restarting port:%d\n",
1520 static int mlx4_en_set_features(
struct net_device *netdev,
1536 .ndo_open = mlx4_en_open,
1537 .ndo_stop = mlx4_en_close,
1540 .ndo_get_stats = mlx4_en_get_stats,
1541 .ndo_set_rx_mode = mlx4_en_set_multicast,
1542 .ndo_set_mac_address = mlx4_en_set_mac,
1544 .ndo_change_mtu = mlx4_en_change_mtu,
1545 .ndo_tx_timeout = mlx4_en_tx_timeout,
1546 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid,
1547 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid,
1548 #ifdef CONFIG_NET_POLL_CONTROLLER
1549 .ndo_poll_controller = mlx4_en_netpoll,
1551 .ndo_set_features = mlx4_en_set_features,
1552 .ndo_setup_tc = mlx4_en_setup_tc,
1553 #ifdef CONFIG_RFS_ACCEL
1554 .ndo_rx_flow_steer = mlx4_en_filter_rfs,
1578 priv = netdev_priv(dev);
1611 #ifdef CONFIG_MLX4_EN_DCB
1612 if (!mlx4_is_slave(priv->
mdev->dev))
1618 priv->
mac = mdev->
dev->caps.def_mac[priv->
port];
1620 en_err(priv,
"Port: %d, invalid mac burned: 0x%llx, quiting\n",
1636 en_err(priv,
"Failed to allocate page for rx qps\n");
1647 netif_set_real_num_rx_queues(dev, priv->
rx_ring_num);
1673 if (mdev->
dev->caps.steering_mode ==
1682 en_err(priv,
"Netdev registration failed for port %d\n", port);
1697 en_err(priv,
"Failed setting port general configurations "
1698 "for port %d, with error %d\n", priv->
port, err);
1703 en_warn(priv,
"Initializing port\n");
1706 en_err(priv,
"Failed Initializing port\n");
1709 mlx4_en_set_default_moderation(priv);