Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
ixgbe_main.c
Go to the documentation of this file.
1 /*******************************************************************************
2 
3  Intel 10 Gigabit PCI Express Linux driver
4  Copyright(c) 1999 - 2012 Intel Corporation.
5 
6  This program is free software; you can redistribute it and/or modify it
7  under the terms and conditions of the GNU General Public License,
8  version 2, as published by the Free Software Foundation.
9 
10  This program is distributed in the hope it will be useful, but WITHOUT
11  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13  more details.
14 
15  You should have received a copy of the GNU General Public License along with
16  this program; if not, write to the Free Software Foundation, Inc.,
17  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 
19  The full GNU General Public License is included in this distribution in
20  the file called "COPYING".
21 
22  Contact Information:
23  e1000-devel Mailing List <[email protected]>
24  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25 
26 *******************************************************************************/
27 
28 #include <linux/types.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/netdevice.h>
32 #include <linux/vmalloc.h>
33 #include <linux/string.h>
34 #include <linux/in.h>
35 #include <linux/interrupt.h>
36 #include <linux/ip.h>
37 #include <linux/tcp.h>
38 #include <linux/sctp.h>
39 #include <linux/pkt_sched.h>
40 #include <linux/ipv6.h>
41 #include <linux/slab.h>
42 #include <net/checksum.h>
43 #include <net/ip6_checksum.h>
44 #include <linux/ethtool.h>
45 #include <linux/if.h>
46 #include <linux/if_vlan.h>
47 #include <linux/prefetch.h>
48 #include <scsi/fc/fc_fcoe.h>
49 
50 #include "ixgbe.h"
51 #include "ixgbe_common.h"
52 #include "ixgbe_dcb_82599.h"
53 #include "ixgbe_sriov.h"
54 
55 char ixgbe_driver_name[] = "ixgbe";
56 static const char ixgbe_driver_string[] =
57  "Intel(R) 10 Gigabit PCI Express Network Driver";
58 #ifdef IXGBE_FCOE
59 char ixgbe_default_device_descr[] =
60  "Intel(R) 10 Gigabit Network Connection";
61 #else
62 static char ixgbe_default_device_descr[] =
63  "Intel(R) 10 Gigabit Network Connection";
64 #endif
65 #define MAJ 3
66 #define MIN 9
67 #define BUILD 15
68 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
69  __stringify(BUILD) "-k"
71 static const char ixgbe_copyright[] =
72  "Copyright (c) 1999-2012 Intel Corporation.";
73 
74 static const struct ixgbe_info *ixgbe_info_tbl[] = {
78 };
79 
80 /* ixgbe_pci_tbl - PCI Device ID Table
81  *
82  * Wildcard entries (PCI_ANY_ID) should come last
83  * Last entry must be all 0s
84  *
85  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
86  * Class, Class Mask, private data (not used) }
87  */
88 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
118  /* required last entry */
119  {0, }
120 };
121 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
122 
123 #ifdef CONFIG_IXGBE_DCA
124 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
125  void *p);
126 static struct notifier_block dca_notifier = {
127  .notifier_call = ixgbe_notify_dca,
128  .next = NULL,
129  .priority = 0
130 };
131 #endif
132 
133 #ifdef CONFIG_PCI_IOV
134 static unsigned int max_vfs;
135 module_param(max_vfs, uint, 0);
136 MODULE_PARM_DESC(max_vfs,
137  "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63");
138 #endif /* CONFIG_PCI_IOV */
139 
140 static unsigned int allow_unsupported_sfp;
141 module_param(allow_unsupported_sfp, uint, 0);
142 MODULE_PARM_DESC(allow_unsupported_sfp,
143  "Allow unsupported and untested SFP+ modules on 82599-based adapters");
144 
145 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
146 static int debug = -1;
147 module_param(debug, int, 0);
148 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
149 
150 MODULE_AUTHOR("Intel Corporation, <[email protected]>");
151 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
152 MODULE_LICENSE("GPL");
154 
155 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
156 {
157  if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
159  schedule_work(&adapter->service_task);
160 }
161 
162 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
163 {
165 
166  /* flush memory to make sure state is correct before next watchdog */
169 }
170 
173  char *name;
174 };
175 
176 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
177 
178  /* General Registers */
179  {IXGBE_CTRL, "CTRL"},
180  {IXGBE_STATUS, "STATUS"},
181  {IXGBE_CTRL_EXT, "CTRL_EXT"},
182 
183  /* Interrupt Registers */
184  {IXGBE_EICR, "EICR"},
185 
186  /* RX Registers */
187  {IXGBE_SRRCTL(0), "SRRCTL"},
188  {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
189  {IXGBE_RDLEN(0), "RDLEN"},
190  {IXGBE_RDH(0), "RDH"},
191  {IXGBE_RDT(0), "RDT"},
192  {IXGBE_RXDCTL(0), "RXDCTL"},
193  {IXGBE_RDBAL(0), "RDBAL"},
194  {IXGBE_RDBAH(0), "RDBAH"},
195 
196  /* TX Registers */
197  {IXGBE_TDBAL(0), "TDBAL"},
198  {IXGBE_TDBAH(0), "TDBAH"},
199  {IXGBE_TDLEN(0), "TDLEN"},
200  {IXGBE_TDH(0), "TDH"},
201  {IXGBE_TDT(0), "TDT"},
202  {IXGBE_TXDCTL(0), "TXDCTL"},
203 
204  /* List Terminator */
205  {}
206 };
207 
208 
209 /*
210  * ixgbe_regdump - register printout routine
211  */
212 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
213 {
214  int i = 0, j = 0;
215  char rname[16];
216  u32 regs[64];
217 
218  switch (reginfo->ofs) {
219  case IXGBE_SRRCTL(0):
220  for (i = 0; i < 64; i++)
221  regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
222  break;
223  case IXGBE_DCA_RXCTRL(0):
224  for (i = 0; i < 64; i++)
225  regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
226  break;
227  case IXGBE_RDLEN(0):
228  for (i = 0; i < 64; i++)
229  regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
230  break;
231  case IXGBE_RDH(0):
232  for (i = 0; i < 64; i++)
233  regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
234  break;
235  case IXGBE_RDT(0):
236  for (i = 0; i < 64; i++)
237  regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
238  break;
239  case IXGBE_RXDCTL(0):
240  for (i = 0; i < 64; i++)
241  regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
242  break;
243  case IXGBE_RDBAL(0):
244  for (i = 0; i < 64; i++)
245  regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
246  break;
247  case IXGBE_RDBAH(0):
248  for (i = 0; i < 64; i++)
249  regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
250  break;
251  case IXGBE_TDBAL(0):
252  for (i = 0; i < 64; i++)
253  regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
254  break;
255  case IXGBE_TDBAH(0):
256  for (i = 0; i < 64; i++)
257  regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
258  break;
259  case IXGBE_TDLEN(0):
260  for (i = 0; i < 64; i++)
261  regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
262  break;
263  case IXGBE_TDH(0):
264  for (i = 0; i < 64; i++)
265  regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
266  break;
267  case IXGBE_TDT(0):
268  for (i = 0; i < 64; i++)
269  regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
270  break;
271  case IXGBE_TXDCTL(0):
272  for (i = 0; i < 64; i++)
273  regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
274  break;
275  default:
276  pr_info("%-15s %08x\n", reginfo->name,
277  IXGBE_READ_REG(hw, reginfo->ofs));
278  return;
279  }
280 
281  for (i = 0; i < 8; i++) {
282  snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
283  pr_err("%-15s", rname);
284  for (j = 0; j < 8; j++)
285  pr_cont(" %08x", regs[i*8+j]);
286  pr_cont("\n");
287  }
288 
289 }
290 
291 /*
292  * ixgbe_dump - Print registers, tx-rings and rx-rings
293  */
294 static void ixgbe_dump(struct ixgbe_adapter *adapter)
295 {
296  struct net_device *netdev = adapter->netdev;
297  struct ixgbe_hw *hw = &adapter->hw;
298  struct ixgbe_reg_info *reginfo;
299  int n = 0;
300  struct ixgbe_ring *tx_ring;
301  struct ixgbe_tx_buffer *tx_buffer;
302  union ixgbe_adv_tx_desc *tx_desc;
303  struct my_u0 { u64 a; u64 b; } *u0;
304  struct ixgbe_ring *rx_ring;
305  union ixgbe_adv_rx_desc *rx_desc;
306  struct ixgbe_rx_buffer *rx_buffer_info;
307  u32 staterr;
308  int i = 0;
309 
310  if (!netif_msg_hw(adapter))
311  return;
312 
313  /* Print netdevice Info */
314  if (netdev) {
315  dev_info(&adapter->pdev->dev, "Net device Info\n");
316  pr_info("Device Name state "
317  "trans_start last_rx\n");
318  pr_info("%-15s %016lX %016lX %016lX\n",
319  netdev->name,
320  netdev->state,
321  netdev->trans_start,
322  netdev->last_rx);
323  }
324 
325  /* Print Registers */
326  dev_info(&adapter->pdev->dev, "Register Dump\n");
327  pr_info(" Register Name Value\n");
328  for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
329  reginfo->name; reginfo++) {
330  ixgbe_regdump(hw, reginfo);
331  }
332 
333  /* Print TX Ring Summary */
334  if (!netdev || !netif_running(netdev))
335  goto exit;
336 
337  dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
338  pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
339  for (n = 0; n < adapter->num_tx_queues; n++) {
340  tx_ring = adapter->tx_ring[n];
341  tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
342  pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
343  n, tx_ring->next_to_use, tx_ring->next_to_clean,
344  (u64)dma_unmap_addr(tx_buffer, dma),
345  dma_unmap_len(tx_buffer, len),
346  tx_buffer->next_to_watch,
347  (u64)tx_buffer->time_stamp);
348  }
349 
350  /* Print TX Rings */
351  if (!netif_msg_tx_done(adapter))
352  goto rx_ring_summary;
353 
354  dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
355 
356  /* Transmit Descriptor Formats
357  *
358  * Advanced Transmit Descriptor
359  * +--------------------------------------------------------------+
360  * 0 | Buffer Address [63:0] |
361  * +--------------------------------------------------------------+
362  * 8 | PAYLEN | PORTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
363  * +--------------------------------------------------------------+
364  * 63 46 45 40 39 36 35 32 31 24 23 20 19 0
365  */
366 
367  for (n = 0; n < adapter->num_tx_queues; n++) {
368  tx_ring = adapter->tx_ring[n];
369  pr_info("------------------------------------\n");
370  pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
371  pr_info("------------------------------------\n");
372  pr_info("T [desc] [address 63:0 ] "
373  "[PlPOIdStDDt Ln] [bi->dma ] "
374  "leng ntw timestamp bi->skb\n");
375 
376  for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
377  tx_desc = IXGBE_TX_DESC(tx_ring, i);
378  tx_buffer = &tx_ring->tx_buffer_info[i];
379  u0 = (struct my_u0 *)tx_desc;
380  pr_info("T [0x%03X] %016llX %016llX %016llX"
381  " %04X %p %016llX %p", i,
382  le64_to_cpu(u0->a),
383  le64_to_cpu(u0->b),
384  (u64)dma_unmap_addr(tx_buffer, dma),
385  dma_unmap_len(tx_buffer, len),
386  tx_buffer->next_to_watch,
387  (u64)tx_buffer->time_stamp,
388  tx_buffer->skb);
389  if (i == tx_ring->next_to_use &&
390  i == tx_ring->next_to_clean)
391  pr_cont(" NTC/U\n");
392  else if (i == tx_ring->next_to_use)
393  pr_cont(" NTU\n");
394  else if (i == tx_ring->next_to_clean)
395  pr_cont(" NTC\n");
396  else
397  pr_cont("\n");
398 
399  if (netif_msg_pktdata(adapter) &&
400  tx_buffer->skb)
401  print_hex_dump(KERN_INFO, "",
402  DUMP_PREFIX_ADDRESS, 16, 1,
403  tx_buffer->skb->data,
404  dma_unmap_len(tx_buffer, len),
405  true);
406  }
407  }
408 
409  /* Print RX Rings Summary */
410 rx_ring_summary:
411  dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
412  pr_info("Queue [NTU] [NTC]\n");
413  for (n = 0; n < adapter->num_rx_queues; n++) {
414  rx_ring = adapter->rx_ring[n];
415  pr_info("%5d %5X %5X\n",
416  n, rx_ring->next_to_use, rx_ring->next_to_clean);
417  }
418 
419  /* Print RX Rings */
420  if (!netif_msg_rx_status(adapter))
421  goto exit;
422 
423  dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
424 
425  /* Advanced Receive Descriptor (Read) Format
426  * 63 1 0
427  * +-----------------------------------------------------+
428  * 0 | Packet Buffer Address [63:1] |A0/NSE|
429  * +----------------------------------------------+------+
430  * 8 | Header Buffer Address [63:1] | DD |
431  * +-----------------------------------------------------+
432  *
433  *
434  * Advanced Receive Descriptor (Write-Back) Format
435  *
436  * 63 48 47 32 31 30 21 20 16 15 4 3 0
437  * +------------------------------------------------------+
438  * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
439  * | Checksum Ident | | | | Type | Type |
440  * +------------------------------------------------------+
441  * 8 | VLAN Tag | Length | Extended Error | Extended Status |
442  * +------------------------------------------------------+
443  * 63 48 47 32 31 20 19 0
444  */
445  for (n = 0; n < adapter->num_rx_queues; n++) {
446  rx_ring = adapter->rx_ring[n];
447  pr_info("------------------------------------\n");
448  pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
449  pr_info("------------------------------------\n");
450  pr_info("R [desc] [ PktBuf A0] "
451  "[ HeadBuf DD] [bi->dma ] [bi->skb] "
452  "<-- Adv Rx Read format\n");
453  pr_info("RWB[desc] [PcsmIpSHl PtRs] "
454  "[vl er S cks ln] ---------------- [bi->skb] "
455  "<-- Adv Rx Write-Back format\n");
456 
457  for (i = 0; i < rx_ring->count; i++) {
458  rx_buffer_info = &rx_ring->rx_buffer_info[i];
459  rx_desc = IXGBE_RX_DESC(rx_ring, i);
460  u0 = (struct my_u0 *)rx_desc;
461  staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
462  if (staterr & IXGBE_RXD_STAT_DD) {
463  /* Descriptor Done */
464  pr_info("RWB[0x%03X] %016llX "
465  "%016llX ---------------- %p", i,
466  le64_to_cpu(u0->a),
467  le64_to_cpu(u0->b),
468  rx_buffer_info->skb);
469  } else {
470  pr_info("R [0x%03X] %016llX "
471  "%016llX %016llX %p", i,
472  le64_to_cpu(u0->a),
473  le64_to_cpu(u0->b),
474  (u64)rx_buffer_info->dma,
475  rx_buffer_info->skb);
476 
477  if (netif_msg_pktdata(adapter) &&
478  rx_buffer_info->dma) {
479  print_hex_dump(KERN_INFO, "",
480  DUMP_PREFIX_ADDRESS, 16, 1,
481  page_address(rx_buffer_info->page) +
482  rx_buffer_info->page_offset,
483  ixgbe_rx_bufsz(rx_ring), true);
484  }
485  }
486 
487  if (i == rx_ring->next_to_use)
488  pr_cont(" NTU\n");
489  else if (i == rx_ring->next_to_clean)
490  pr_cont(" NTC\n");
491  else
492  pr_cont("\n");
493 
494  }
495  }
496 
497 exit:
498  return;
499 }
500 
501 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
502 {
503  u32 ctrl_ext;
504 
505  /* Let firmware take over control of h/w */
506  ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
507  IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
508  ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
509 }
510 
511 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
512 {
513  u32 ctrl_ext;
514 
515  /* Let firmware know the driver has taken over */
516  ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
517  IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
518  ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
519 }
520 
529 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
530  u8 queue, u8 msix_vector)
531 {
532  u32 ivar, index;
533  struct ixgbe_hw *hw = &adapter->hw;
534  switch (hw->mac.type) {
535  case ixgbe_mac_82598EB:
536  msix_vector |= IXGBE_IVAR_ALLOC_VAL;
537  if (direction == -1)
538  direction = 0;
539  index = (((direction * 64) + queue) >> 2) & 0x1F;
540  ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
541  ivar &= ~(0xFF << (8 * (queue & 0x3)));
542  ivar |= (msix_vector << (8 * (queue & 0x3)));
543  IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
544  break;
545  case ixgbe_mac_82599EB:
546  case ixgbe_mac_X540:
547  if (direction == -1) {
548  /* other causes */
549  msix_vector |= IXGBE_IVAR_ALLOC_VAL;
550  index = ((queue & 1) * 8);
551  ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
552  ivar &= ~(0xFF << index);
553  ivar |= (msix_vector << index);
554  IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
555  break;
556  } else {
557  /* tx or rx causes */
558  msix_vector |= IXGBE_IVAR_ALLOC_VAL;
559  index = ((16 * (queue & 1)) + (8 * direction));
560  ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
561  ivar &= ~(0xFF << index);
562  ivar |= (msix_vector << index);
563  IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
564  break;
565  }
566  default:
567  break;
568  }
569 }
570 
571 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
572  u64 qmask)
573 {
574  u32 mask;
575 
576  switch (adapter->hw.mac.type) {
577  case ixgbe_mac_82598EB:
578  mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
579  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
580  break;
581  case ixgbe_mac_82599EB:
582  case ixgbe_mac_X540:
583  mask = (qmask & 0xFFFFFFFF);
584  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
585  mask = (qmask >> 32);
586  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
587  break;
588  default:
589  break;
590  }
591 }
592 
594  struct ixgbe_tx_buffer *tx_buffer)
595 {
596  if (tx_buffer->skb) {
597  dev_kfree_skb_any(tx_buffer->skb);
598  if (dma_unmap_len(tx_buffer, len))
599  dma_unmap_single(ring->dev,
600  dma_unmap_addr(tx_buffer, dma),
601  dma_unmap_len(tx_buffer, len),
602  DMA_TO_DEVICE);
603  } else if (dma_unmap_len(tx_buffer, len)) {
604  dma_unmap_page(ring->dev,
605  dma_unmap_addr(tx_buffer, dma),
606  dma_unmap_len(tx_buffer, len),
607  DMA_TO_DEVICE);
608  }
609  tx_buffer->next_to_watch = NULL;
610  tx_buffer->skb = NULL;
611  dma_unmap_len_set(tx_buffer, len, 0);
612  /* tx_buffer must be completely set up in the transmit path */
613 }
614 
615 static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
616 {
617  struct ixgbe_hw *hw = &adapter->hw;
618  struct ixgbe_hw_stats *hwstats = &adapter->stats;
619  int i;
620  u32 data;
621 
622  if ((hw->fc.current_mode != ixgbe_fc_full) &&
623  (hw->fc.current_mode != ixgbe_fc_rx_pause))
624  return;
625 
626  switch (hw->mac.type) {
627  case ixgbe_mac_82598EB:
628  data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
629  break;
630  default:
631  data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
632  }
633  hwstats->lxoffrxc += data;
634 
635  /* refill credits (no tx hang) if we received xoff */
636  if (!data)
637  return;
638 
639  for (i = 0; i < adapter->num_tx_queues; i++)
641  &adapter->tx_ring[i]->state);
642 }
643 
644 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
645 {
646  struct ixgbe_hw *hw = &adapter->hw;
647  struct ixgbe_hw_stats *hwstats = &adapter->stats;
648  u32 xoff[8] = {0};
649  int i;
650  bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
651 
652  if (adapter->ixgbe_ieee_pfc)
653  pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
654 
655  if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
656  ixgbe_update_xoff_rx_lfc(adapter);
657  return;
658  }
659 
660  /* update stats for each tc, only valid with PFC enabled */
661  for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
662  switch (hw->mac.type) {
663  case ixgbe_mac_82598EB:
664  xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
665  break;
666  default:
667  xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
668  }
669  hwstats->pxoffrxc[i] += xoff[i];
670  }
671 
672  /* disarm tx queues that have received xoff frames */
673  for (i = 0; i < adapter->num_tx_queues; i++) {
674  struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
675  u8 tc = tx_ring->dcb_tc;
676 
677  if (xoff[tc])
679  }
680 }
681 
682 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
683 {
684  return ring->stats.packets;
685 }
686 
687 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
688 {
689  struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
690  struct ixgbe_hw *hw = &adapter->hw;
691 
692  u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
693  u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
694 
695  if (head != tail)
696  return (head < tail) ?
697  tail - head : (tail + ring->count - head);
698 
699  return 0;
700 }
701 
702 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
703 {
704  u32 tx_done = ixgbe_get_tx_completed(tx_ring);
705  u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
706  u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
707  bool ret = false;
708 
709  clear_check_for_tx_hang(tx_ring);
710 
711  /*
712  * Check for a hung queue, but be thorough. This verifies
713  * that a transmit has been completed since the previous
714  * check AND there is at least one packet pending. The
715  * ARMED bit is set to indicate a potential hang. The
716  * bit is cleared if a pause frame is received to remove
717  * false hang detection due to PFC or 802.3x frames. By
718  * requiring this to fail twice we avoid races with
719  * pfc clearing the ARMED bit and conditions where we
720  * run the check_tx_hang logic with a transmit completion
721  * pending but without time to complete it yet.
722  */
723  if ((tx_done_old == tx_done) && tx_pending) {
724  /* make sure it is true for two checks in a row */
726  &tx_ring->state);
727  } else {
728  /* update completed stats and continue */
729  tx_ring->tx_stats.tx_done_old = tx_done;
730  /* reset the countdown */
732  }
733 
734  return ret;
735 }
736 
741 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
742 {
743 
744  /* Do the reset outside of interrupt context */
745  if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
747  ixgbe_service_event_schedule(adapter);
748  }
749 }
750 
756 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
757  struct ixgbe_ring *tx_ring)
758 {
759  struct ixgbe_adapter *adapter = q_vector->adapter;
760  struct ixgbe_tx_buffer *tx_buffer;
761  union ixgbe_adv_tx_desc *tx_desc;
762  unsigned int total_bytes = 0, total_packets = 0;
763  unsigned int budget = q_vector->tx.work_limit;
764  unsigned int i = tx_ring->next_to_clean;
765 
766  if (test_bit(__IXGBE_DOWN, &adapter->state))
767  return true;
768 
769  tx_buffer = &tx_ring->tx_buffer_info[i];
770  tx_desc = IXGBE_TX_DESC(tx_ring, i);
771  i -= tx_ring->count;
772 
773  do {
774  union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
775 
776  /* if next_to_watch is not set then there is no work pending */
777  if (!eop_desc)
778  break;
779 
780  /* prevent any other reads prior to eop_desc */
781  rmb();
782 
783  /* if DD is not set pending work has not been completed */
784  if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
785  break;
786 
787  /* clear next_to_watch to prevent false hangs */
788  tx_buffer->next_to_watch = NULL;
789 
790  /* update the statistics for this packet */
791  total_bytes += tx_buffer->bytecount;
792  total_packets += tx_buffer->gso_segs;
793 
794 #ifdef CONFIG_IXGBE_PTP
795  if (unlikely(tx_buffer->tx_flags & IXGBE_TX_FLAGS_TSTAMP))
796  ixgbe_ptp_tx_hwtstamp(q_vector, tx_buffer->skb);
797 #endif
798 
799  /* free the skb */
800  dev_kfree_skb_any(tx_buffer->skb);
801 
802  /* unmap skb header data */
803  dma_unmap_single(tx_ring->dev,
804  dma_unmap_addr(tx_buffer, dma),
805  dma_unmap_len(tx_buffer, len),
806  DMA_TO_DEVICE);
807 
808  /* clear tx_buffer data */
809  tx_buffer->skb = NULL;
810  dma_unmap_len_set(tx_buffer, len, 0);
811 
812  /* unmap remaining buffers */
813  while (tx_desc != eop_desc) {
814  tx_buffer++;
815  tx_desc++;
816  i++;
817  if (unlikely(!i)) {
818  i -= tx_ring->count;
819  tx_buffer = tx_ring->tx_buffer_info;
820  tx_desc = IXGBE_TX_DESC(tx_ring, 0);
821  }
822 
823  /* unmap any remaining paged data */
824  if (dma_unmap_len(tx_buffer, len)) {
825  dma_unmap_page(tx_ring->dev,
826  dma_unmap_addr(tx_buffer, dma),
827  dma_unmap_len(tx_buffer, len),
828  DMA_TO_DEVICE);
829  dma_unmap_len_set(tx_buffer, len, 0);
830  }
831  }
832 
833  /* move us one more past the eop_desc for start of next pkt */
834  tx_buffer++;
835  tx_desc++;
836  i++;
837  if (unlikely(!i)) {
838  i -= tx_ring->count;
839  tx_buffer = tx_ring->tx_buffer_info;
840  tx_desc = IXGBE_TX_DESC(tx_ring, 0);
841  }
842 
843  /* issue prefetch for next Tx descriptor */
844  prefetch(tx_desc);
845 
846  /* update budget accounting */
847  budget--;
848  } while (likely(budget));
849 
850  i += tx_ring->count;
851  tx_ring->next_to_clean = i;
852  u64_stats_update_begin(&tx_ring->syncp);
853  tx_ring->stats.bytes += total_bytes;
854  tx_ring->stats.packets += total_packets;
855  u64_stats_update_end(&tx_ring->syncp);
856  q_vector->tx.total_bytes += total_bytes;
857  q_vector->tx.total_packets += total_packets;
858 
859  if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
860  /* schedule immediate reset if we believe we hung */
861  struct ixgbe_hw *hw = &adapter->hw;
862  e_err(drv, "Detected Tx Unit Hang\n"
863  " Tx Queue <%d>\n"
864  " TDH, TDT <%x>, <%x>\n"
865  " next_to_use <%x>\n"
866  " next_to_clean <%x>\n"
867  "tx_buffer_info[next_to_clean]\n"
868  " time_stamp <%lx>\n"
869  " jiffies <%lx>\n",
870  tx_ring->queue_index,
871  IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
872  IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
873  tx_ring->next_to_use, i,
874  tx_ring->tx_buffer_info[i].time_stamp, jiffies);
875 
876  netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
877 
878  e_info(probe,
879  "tx hang %d detected on queue %d, resetting adapter\n",
880  adapter->tx_timeout_count + 1, tx_ring->queue_index);
881 
882  /* schedule immediate reset if we believe we hung */
883  ixgbe_tx_timeout_reset(adapter);
884 
885  /* the adapter is about to reset, no point in enabling stuff */
886  return true;
887  }
888 
889  netdev_tx_completed_queue(txring_txq(tx_ring),
890  total_packets, total_bytes);
891 
892 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
893  if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
894  (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
895  /* Make sure that anybody stopping the queue after this
896  * sees the new next_to_clean.
897  */
898  smp_mb();
899  if (__netif_subqueue_stopped(tx_ring->netdev,
900  tx_ring->queue_index)
901  && !test_bit(__IXGBE_DOWN, &adapter->state)) {
902  netif_wake_subqueue(tx_ring->netdev,
903  tx_ring->queue_index);
904  ++tx_ring->tx_stats.restart_queue;
905  }
906  }
907 
908  return !!budget;
909 }
910 
911 #ifdef CONFIG_IXGBE_DCA
912 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
913  struct ixgbe_ring *tx_ring,
914  int cpu)
915 {
916  struct ixgbe_hw *hw = &adapter->hw;
917  u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
918  u16 reg_offset;
919 
920  switch (hw->mac.type) {
921  case ixgbe_mac_82598EB:
922  reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
923  break;
924  case ixgbe_mac_82599EB:
925  case ixgbe_mac_X540:
926  reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
928  break;
929  default:
930  /* for unknown hardware do not write register */
931  return;
932  }
933 
934  /*
935  * We can enable relaxed ordering for reads, but not writes when
936  * DCA is enabled. This is due to a known issue in some chipsets
937  * which will cause the DCA tag to be cleared.
938  */
939  txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
942 
943  IXGBE_WRITE_REG(hw, reg_offset, txctrl);
944 }
945 
946 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
947  struct ixgbe_ring *rx_ring,
948  int cpu)
949 {
950  struct ixgbe_hw *hw = &adapter->hw;
951  u32 rxctrl = dca3_get_tag(rx_ring->dev, cpu);
952  u8 reg_idx = rx_ring->reg_idx;
953 
954 
955  switch (hw->mac.type) {
956  case ixgbe_mac_82599EB:
957  case ixgbe_mac_X540:
959  break;
960  default:
961  break;
962  }
963 
964  /*
965  * We can enable relaxed ordering for reads, but not writes when
966  * DCA is enabled. This is due to a known issue in some chipsets
967  * which will cause the DCA tag to be cleared.
968  */
969  rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
972 
973  IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
974 }
975 
976 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
977 {
978  struct ixgbe_adapter *adapter = q_vector->adapter;
979  struct ixgbe_ring *ring;
980  int cpu = get_cpu();
981 
982  if (q_vector->cpu == cpu)
983  goto out_no_update;
984 
985  ixgbe_for_each_ring(ring, q_vector->tx)
986  ixgbe_update_tx_dca(adapter, ring, cpu);
987 
988  ixgbe_for_each_ring(ring, q_vector->rx)
989  ixgbe_update_rx_dca(adapter, ring, cpu);
990 
991  q_vector->cpu = cpu;
992 out_no_update:
993  put_cpu();
994 }
995 
996 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
997 {
998  int i;
999 
1000  if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
1001  return;
1002 
1003  /* always use CB2 mode, difference is masked in the CB driver */
1004  IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
1005 
1006  for (i = 0; i < adapter->num_q_vectors; i++) {
1007  adapter->q_vector[i]->cpu = -1;
1008  ixgbe_update_dca(adapter->q_vector[i]);
1009  }
1010 }
1011 
1012 static int __ixgbe_notify_dca(struct device *dev, void *data)
1013 {
1014  struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1015  unsigned long event = *(unsigned long *)data;
1016 
1017  if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
1018  return 0;
1019 
1020  switch (event) {
1021  case DCA_PROVIDER_ADD:
1022  /* if we're already enabled, don't do it again */
1023  if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1024  break;
1025  if (dca_add_requester(dev) == 0) {
1026  adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1027  ixgbe_setup_dca(adapter);
1028  break;
1029  }
1030  /* Fall Through since DCA is disabled. */
1031  case DCA_PROVIDER_REMOVE:
1032  if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1033  dca_remove_requester(dev);
1034  adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
1035  IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
1036  }
1037  break;
1038  }
1039 
1040  return 0;
1041 }
1042 
1043 #endif /* CONFIG_IXGBE_DCA */
1044 static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
1045  union ixgbe_adv_rx_desc *rx_desc,
1046  struct sk_buff *skb)
1047 {
1048  if (ring->netdev->features & NETIF_F_RXHASH)
1049  skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
1050 }
1051 
1052 #ifdef IXGBE_FCOE
1053 
1060 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
1061  union ixgbe_adv_rx_desc *rx_desc)
1062 {
1063  __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1064 
1065  return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
1069 }
1070 
1071 #endif /* IXGBE_FCOE */
1072 
1078 static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
1079  union ixgbe_adv_rx_desc *rx_desc,
1080  struct sk_buff *skb)
1081 {
1082  skb_checksum_none_assert(skb);
1083 
1084  /* Rx csum disabled */
1085  if (!(ring->netdev->features & NETIF_F_RXCSUM))
1086  return;
1087 
1088  /* if IP and error */
1089  if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
1090  ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
1091  ring->rx_stats.csum_err++;
1092  return;
1093  }
1094 
1095  if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
1096  return;
1097 
1098  if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
1099  __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1100 
1101  /*
1102  * 82599 errata, UDP frames with a 0 checksum can be marked as
1103  * checksum errors.
1104  */
1105  if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
1107  return;
1108 
1109  ring->rx_stats.csum_err++;
1110  return;
1111  }
1112 
1113  /* It must be a TCP or UDP packet with a valid checksum */
1115 }
1116 
1117 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1118 {
1119  rx_ring->next_to_use = val;
1120 
1121  /* update next to alloc since we have filled the ring */
1122  rx_ring->next_to_alloc = val;
1123  /*
1124  * Force memory writes to complete before letting h/w
1125  * know there are new descriptors to fetch. (Only
1126  * applicable for weak-ordered memory model archs,
1127  * such as IA-64).
1128  */
1129  wmb();
1130  writel(val, rx_ring->tail);
1131 }
1132 
1133 static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
1134  struct ixgbe_rx_buffer *bi)
1135 {
1136  struct page *page = bi->page;
1137  dma_addr_t dma = bi->dma;
1138 
1139  /* since we are recycling buffers we should seldom need to alloc */
1140  if (likely(dma))
1141  return true;
1142 
1143  /* alloc new page for storage */
1144  if (likely(!page)) {
1145  page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP,
1146  bi->skb, ixgbe_rx_pg_order(rx_ring));
1147  if (unlikely(!page)) {
1148  rx_ring->rx_stats.alloc_rx_page_failed++;
1149  return false;
1150  }
1151  bi->page = page;
1152  }
1153 
1154  /* map page for use */
1155  dma = dma_map_page(rx_ring->dev, page, 0,
1156  ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
1157 
1158  /*
1159  * if mapping failed free memory back to system since
1160  * there isn't much point in holding memory we can't use
1161  */
1162  if (dma_mapping_error(rx_ring->dev, dma)) {
1163  __free_pages(page, ixgbe_rx_pg_order(rx_ring));
1164  bi->page = NULL;
1165 
1166  rx_ring->rx_stats.alloc_rx_page_failed++;
1167  return false;
1168  }
1169 
1170  bi->dma = dma;
1171  bi->page_offset = 0;
1172 
1173  return true;
1174 }
1175 
1181 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1182 {
1183  union ixgbe_adv_rx_desc *rx_desc;
1184  struct ixgbe_rx_buffer *bi;
1185  u16 i = rx_ring->next_to_use;
1186 
1187  /* nothing to do */
1188  if (!cleaned_count)
1189  return;
1190 
1191  rx_desc = IXGBE_RX_DESC(rx_ring, i);
1192  bi = &rx_ring->rx_buffer_info[i];
1193  i -= rx_ring->count;
1194 
1195  do {
1196  if (!ixgbe_alloc_mapped_page(rx_ring, bi))
1197  break;
1198 
1199  /*
1200  * Refresh the desc even if buffer_addrs didn't change
1201  * because each write-back erases this info.
1202  */
1203  rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1204 
1205  rx_desc++;
1206  bi++;
1207  i++;
1208  if (unlikely(!i)) {
1209  rx_desc = IXGBE_RX_DESC(rx_ring, 0);
1210  bi = rx_ring->rx_buffer_info;
1211  i -= rx_ring->count;
1212  }
1213 
1214  /* clear the hdr_addr for the next_to_use descriptor */
1215  rx_desc->read.hdr_addr = 0;
1216 
1217  cleaned_count--;
1218  } while (cleaned_count);
1219 
1220  i += rx_ring->count;
1221 
1222  if (rx_ring->next_to_use != i)
1223  ixgbe_release_rx_desc(rx_ring, i);
1224 }
1225 
1237 static unsigned int ixgbe_get_headlen(unsigned char *data,
1238  unsigned int max_len)
1239 {
1240  union {
1241  unsigned char *network;
1242  /* l2 headers */
1243  struct ethhdr *eth;
1244  struct vlan_hdr *vlan;
1245  /* l3 headers */
1246  struct iphdr *ipv4;
1247  } hdr;
1248  __be16 protocol;
1249  u8 nexthdr = 0; /* default to not TCP */
1250  u8 hlen;
1251 
1252  /* this should never happen, but better safe than sorry */
1253  if (max_len < ETH_HLEN)
1254  return max_len;
1255 
1256  /* initialize network frame pointer */
1257  hdr.network = data;
1258 
1259  /* set first protocol and move network header forward */
1260  protocol = hdr.eth->h_proto;
1261  hdr.network += ETH_HLEN;
1262 
1263  /* handle any vlan tag if present */
1264  if (protocol == __constant_htons(ETH_P_8021Q)) {
1265  if ((hdr.network - data) > (max_len - VLAN_HLEN))
1266  return max_len;
1267 
1268  protocol = hdr.vlan->h_vlan_encapsulated_proto;
1269  hdr.network += VLAN_HLEN;
1270  }
1271 
1272  /* handle L3 protocols */
1273  if (protocol == __constant_htons(ETH_P_IP)) {
1274  if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
1275  return max_len;
1276 
1277  /* access ihl as a u8 to avoid unaligned access on ia64 */
1278  hlen = (hdr.network[0] & 0x0F) << 2;
1279 
1280  /* verify hlen meets minimum size requirements */
1281  if (hlen < sizeof(struct iphdr))
1282  return hdr.network - data;
1283 
1284  /* record next protocol */
1285  nexthdr = hdr.ipv4->protocol;
1286  hdr.network += hlen;
1287 #ifdef IXGBE_FCOE
1288  } else if (protocol == __constant_htons(ETH_P_FCOE)) {
1289  if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN))
1290  return max_len;
1291  hdr.network += FCOE_HEADER_LEN;
1292 #endif
1293  } else {
1294  return hdr.network - data;
1295  }
1296 
1297  /* finally sort out TCP */
1298  if (nexthdr == IPPROTO_TCP) {
1299  if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
1300  return max_len;
1301 
1302  /* access doff as a u8 to avoid unaligned access on ia64 */
1303  hlen = (hdr.network[12] & 0xF0) >> 2;
1304 
1305  /* verify hlen meets minimum size requirements */
1306  if (hlen < sizeof(struct tcphdr))
1307  return hdr.network - data;
1308 
1309  hdr.network += hlen;
1310  }
1311 
1312  /*
1313  * If everything has gone correctly hdr.network should be the
1314  * data section of the packet and will be the end of the header.
1315  * If not then it probably represents the end of the last recognized
1316  * header.
1317  */
1318  if ((hdr.network - data) < max_len)
1319  return hdr.network - data;
1320  else
1321  return max_len;
1322 }
1323 
1324 static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
1325  struct sk_buff *skb)
1326 {
1327  u16 hdr_len = skb_headlen(skb);
1328 
1329  /* set gso_size to avoid messing up TCP MSS */
1330  skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
1331  IXGBE_CB(skb)->append_cnt);
1332 }
1333 
1334 static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
1335  struct sk_buff *skb)
1336 {
1337  /* if append_cnt is 0 then frame is not RSC */
1338  if (!IXGBE_CB(skb)->append_cnt)
1339  return;
1340 
1341  rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
1342  rx_ring->rx_stats.rsc_flush++;
1343 
1344  ixgbe_set_rsc_gso_size(rx_ring, skb);
1345 
1346  /* gso_size is computed using append_cnt so always clear it last */
1347  IXGBE_CB(skb)->append_cnt = 0;
1348 }
1349 
1360 static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
1361  union ixgbe_adv_rx_desc *rx_desc,
1362  struct sk_buff *skb)
1363 {
1364  struct net_device *dev = rx_ring->netdev;
1365 
1366  ixgbe_update_rsc_stats(rx_ring, skb);
1367 
1368  ixgbe_rx_hash(rx_ring, rx_desc, skb);
1369 
1370  ixgbe_rx_checksum(rx_ring, rx_desc, skb);
1371 
1372 #ifdef CONFIG_IXGBE_PTP
1373  ixgbe_ptp_rx_hwtstamp(rx_ring->q_vector, rx_desc, skb);
1374 #endif
1375 
1376  if ((dev->features & NETIF_F_HW_VLAN_RX) &&
1377  ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
1378  u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1379  __vlan_hwaccel_put_tag(skb, vid);
1380  }
1381 
1382  skb_record_rx_queue(skb, rx_ring->queue_index);
1383 
1384  skb->protocol = eth_type_trans(skb, dev);
1385 }
1386 
1387 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
1388  struct sk_buff *skb)
1389 {
1390  struct ixgbe_adapter *adapter = q_vector->adapter;
1391 
1392  if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1393  napi_gro_receive(&q_vector->napi, skb);
1394  else
1395  netif_rx(skb);
1396 }
1397 
1409 static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
1410  union ixgbe_adv_rx_desc *rx_desc,
1411  struct sk_buff *skb)
1412 {
1413  u32 ntc = rx_ring->next_to_clean + 1;
1414 
1415  /* fetch, update, and store next to clean */
1416  ntc = (ntc < rx_ring->count) ? ntc : 0;
1417  rx_ring->next_to_clean = ntc;
1418 
1419  prefetch(IXGBE_RX_DESC(rx_ring, ntc));
1420 
1421  /* update RSC append count if present */
1422  if (ring_is_rsc_enabled(rx_ring)) {
1423  __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
1425 
1426  if (unlikely(rsc_enabled)) {
1427  u32 rsc_cnt = le32_to_cpu(rsc_enabled);
1428 
1429  rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
1430  IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
1431 
1432  /* update ntc based on RSC value */
1433  ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
1434  ntc &= IXGBE_RXDADV_NEXTP_MASK;
1435  ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
1436  }
1437  }
1438 
1439  /* if we are the last buffer then there is nothing else to do */
1440  if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1441  return false;
1442 
1443  /* place skb in next buffer to be received */
1444  rx_ring->rx_buffer_info[ntc].skb = skb;
1445  rx_ring->rx_stats.non_eop_descs++;
1446 
1447  return true;
1448 }
1449 
1462 static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
1463  struct sk_buff *skb)
1464 {
1465  struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1466  unsigned char *va;
1467  unsigned int pull_len;
1468 
1469  /*
1470  * it is valid to use page_address instead of kmap since we are
1471  * working with pages allocated out of the lomem pool per
1472  * alloc_page(GFP_ATOMIC)
1473  */
1474  va = skb_frag_address(frag);
1475 
1476  /*
1477  * we need the header to contain the greater of either ETH_HLEN or
1478  * 60 bytes if the skb->len is less than 60 for skb_pad.
1479  */
1480  pull_len = ixgbe_get_headlen(va, IXGBE_RX_HDR_SIZE);
1481 
1482  /* align pull length to size of long to optimize memcpy performance */
1483  skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
1484 
1485  /* update all of the pointers */
1486  skb_frag_size_sub(frag, pull_len);
1487  frag->page_offset += pull_len;
1488  skb->data_len -= pull_len;
1489  skb->tail += pull_len;
1490 }
1491 
1502 static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
1503  struct sk_buff *skb)
1504 {
1505  /* if the page was released unmap it, else just sync our portion */
1506  if (unlikely(IXGBE_CB(skb)->page_released)) {
1507  dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma,
1508  ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
1509  IXGBE_CB(skb)->page_released = false;
1510  } else {
1511  struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1512 
1514  IXGBE_CB(skb)->dma,
1515  frag->page_offset,
1516  ixgbe_rx_bufsz(rx_ring),
1517  DMA_FROM_DEVICE);
1518  }
1519  IXGBE_CB(skb)->dma = 0;
1520 }
1521 
1540 static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
1541  union ixgbe_adv_rx_desc *rx_desc,
1542  struct sk_buff *skb)
1543 {
1544  struct net_device *netdev = rx_ring->netdev;
1545 
1546  /* verify that the packet does not have any known errors */
1547  if (unlikely(ixgbe_test_staterr(rx_desc,
1549  !(netdev->features & NETIF_F_RXALL))) {
1550  dev_kfree_skb_any(skb);
1551  return true;
1552  }
1553 
1554  /* place header in linear portion of buffer */
1555  if (skb_is_nonlinear(skb))
1556  ixgbe_pull_tail(rx_ring, skb);
1557 
1558 #ifdef IXGBE_FCOE
1559  /* do not attempt to pad FCoE Frames as this will disrupt DDP */
1560  if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
1561  return false;
1562 
1563 #endif
1564  /* if skb_pad returns an error the skb was freed */
1565  if (unlikely(skb->len < 60)) {
1566  int pad_len = 60 - skb->len;
1567 
1568  if (skb_pad(skb, pad_len))
1569  return true;
1570  __skb_put(skb, pad_len);
1571  }
1572 
1573  return false;
1574 }
1575 
1583 static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
1584  struct ixgbe_rx_buffer *old_buff)
1585 {
1586  struct ixgbe_rx_buffer *new_buff;
1587  u16 nta = rx_ring->next_to_alloc;
1588 
1589  new_buff = &rx_ring->rx_buffer_info[nta];
1590 
1591  /* update, and store next to alloc */
1592  nta++;
1593  rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1594 
1595  /* transfer page from old buffer to new buffer */
1596  new_buff->page = old_buff->page;
1597  new_buff->dma = old_buff->dma;
1598  new_buff->page_offset = old_buff->page_offset;
1599 
1600  /* sync the buffer for use by the device */
1601  dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
1602  new_buff->page_offset,
1603  ixgbe_rx_bufsz(rx_ring),
1604  DMA_FROM_DEVICE);
1605 }
1606 
1622 static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
1623  struct ixgbe_rx_buffer *rx_buffer,
1624  union ixgbe_adv_rx_desc *rx_desc,
1625  struct sk_buff *skb)
1626 {
1627  struct page *page = rx_buffer->page;
1628  unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
1629 #if (PAGE_SIZE < 8192)
1630  unsigned int truesize = ixgbe_rx_bufsz(rx_ring);
1631 #else
1632  unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
1633  unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) -
1634  ixgbe_rx_bufsz(rx_ring);
1635 #endif
1636 
1637  if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
1638  unsigned char *va = page_address(page) + rx_buffer->page_offset;
1639 
1640  memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
1641 
1642  /* we can reuse buffer as-is, just make sure it is local */
1643  if (likely(page_to_nid(page) == numa_node_id()))
1644  return true;
1645 
1646  /* this page cannot be reused so discard it */
1647  put_page(page);
1648  return false;
1649  }
1650 
1651  skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
1652  rx_buffer->page_offset, size, truesize);
1653 
1654  /* avoid re-using remote pages */
1655  if (unlikely(page_to_nid(page) != numa_node_id()))
1656  return false;
1657 
1658 #if (PAGE_SIZE < 8192)
1659  /* if we are only owner of page we can reuse it */
1660  if (unlikely(page_count(page) != 1))
1661  return false;
1662 
1663  /* flip page offset to other buffer */
1664  rx_buffer->page_offset ^= truesize;
1665 
1666  /*
1667  * since we are the only owner of the page and we need to
1668  * increment it, just set the value to 2 in order to avoid
1669  * an unecessary locked operation
1670  */
1671  atomic_set(&page->_count, 2);
1672 #else
1673  /* move offset up to the next cache line */
1674  rx_buffer->page_offset += truesize;
1675 
1676  if (rx_buffer->page_offset > last_offset)
1677  return false;
1678 
1679  /* bump ref count on page before it is given to the stack */
1680  get_page(page);
1681 #endif
1682 
1683  return true;
1684 }
1685 
1686 static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring,
1687  union ixgbe_adv_rx_desc *rx_desc)
1688 {
1689  struct ixgbe_rx_buffer *rx_buffer;
1690  struct sk_buff *skb;
1691  struct page *page;
1692 
1693  rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1694  page = rx_buffer->page;
1695  prefetchw(page);
1696 
1697  skb = rx_buffer->skb;
1698 
1699  if (likely(!skb)) {
1700  void *page_addr = page_address(page) +
1701  rx_buffer->page_offset;
1702 
1703  /* prefetch first cache line of first page */
1704  prefetch(page_addr);
1705 #if L1_CACHE_BYTES < 128
1706  prefetch(page_addr + L1_CACHE_BYTES);
1707 #endif
1708 
1709  /* allocate a skb to store the frags */
1710  skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1712  if (unlikely(!skb)) {
1713  rx_ring->rx_stats.alloc_rx_buff_failed++;
1714  return NULL;
1715  }
1716 
1717  /*
1718  * we will be copying header into skb->data in
1719  * pskb_may_pull so it is in our interest to prefetch
1720  * it now to avoid a possible cache miss
1721  */
1722  prefetchw(skb->data);
1723 
1724  /*
1725  * Delay unmapping of the first packet. It carries the
1726  * header information, HW may still access the header
1727  * after the writeback. Only unmap it when EOP is
1728  * reached
1729  */
1730  if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1731  goto dma_sync;
1732 
1733  IXGBE_CB(skb)->dma = rx_buffer->dma;
1734  } else {
1735  if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
1736  ixgbe_dma_sync_frag(rx_ring, skb);
1737 
1738 dma_sync:
1739  /* we are reusing so sync this buffer for CPU use */
1741  rx_buffer->dma,
1742  rx_buffer->page_offset,
1743  ixgbe_rx_bufsz(rx_ring),
1744  DMA_FROM_DEVICE);
1745  }
1746 
1747  /* pull page into skb */
1748  if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
1749  /* hand second half of page back to the ring */
1750  ixgbe_reuse_rx_page(rx_ring, rx_buffer);
1751  } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) {
1752  /* the page has been released from the ring */
1753  IXGBE_CB(skb)->page_released = true;
1754  } else {
1755  /* we are not reusing the buffer so unmap it */
1756  dma_unmap_page(rx_ring->dev, rx_buffer->dma,
1757  ixgbe_rx_pg_size(rx_ring),
1758  DMA_FROM_DEVICE);
1759  }
1760 
1761  /* clear contents of buffer_info */
1762  rx_buffer->skb = NULL;
1763  rx_buffer->dma = 0;
1764  rx_buffer->page = NULL;
1765 
1766  return skb;
1767 }
1768 
1782 static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1783  struct ixgbe_ring *rx_ring,
1784  int budget)
1785 {
1786  unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1787 #ifdef IXGBE_FCOE
1788  struct ixgbe_adapter *adapter = q_vector->adapter;
1789  int ddp_bytes;
1790  unsigned int mss = 0;
1791 #endif /* IXGBE_FCOE */
1792  u16 cleaned_count = ixgbe_desc_unused(rx_ring);
1793 
1794  do {
1795  union ixgbe_adv_rx_desc *rx_desc;
1796  struct sk_buff *skb;
1797 
1798  /* return some buffers to hardware, one at a time is too slow */
1799  if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1800  ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1801  cleaned_count = 0;
1802  }
1803 
1804  rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
1805 
1806  if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
1807  break;
1808 
1809  /*
1810  * This memory barrier is needed to keep us from reading
1811  * any other fields out of the rx_desc until we know the
1812  * RXD_STAT_DD bit is set
1813  */
1814  rmb();
1815 
1816  /* retrieve a buffer from the ring */
1817  skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc);
1818 
1819  /* exit if we failed to retrieve a buffer */
1820  if (!skb)
1821  break;
1822 
1823  cleaned_count++;
1824 
1825  /* place incomplete frames back on ring for completion */
1826  if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
1827  continue;
1828 
1829  /* verify the packet layout is correct */
1830  if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
1831  continue;
1832 
1833  /* probably a little skewed due to removing CRC */
1834  total_rx_bytes += skb->len;
1835  total_rx_packets++;
1836 
1837  /* populate checksum, timestamp, VLAN, and protocol */
1838  ixgbe_process_skb_fields(rx_ring, rx_desc, skb);
1839 
1840 #ifdef IXGBE_FCOE
1841  /* if ddp, not passing to ULD unless for FCP_RSP or error */
1842  if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
1843  ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
1844  /* include DDPed FCoE data */
1845  if (ddp_bytes > 0) {
1846  if (!mss) {
1847  mss = rx_ring->netdev->mtu -
1848  sizeof(struct fcoe_hdr) -
1851  if (mss > 512)
1852  mss &= ~511;
1853  }
1854  total_rx_bytes += ddp_bytes;
1855  total_rx_packets += DIV_ROUND_UP(ddp_bytes,
1856  mss);
1857  }
1858  if (!ddp_bytes) {
1859  dev_kfree_skb_any(skb);
1860  continue;
1861  }
1862  }
1863 
1864 #endif /* IXGBE_FCOE */
1865  ixgbe_rx_skb(q_vector, skb);
1866 
1867  /* update budget accounting */
1868  budget--;
1869  } while (likely(budget));
1870 
1871  u64_stats_update_begin(&rx_ring->syncp);
1872  rx_ring->stats.packets += total_rx_packets;
1873  rx_ring->stats.bytes += total_rx_bytes;
1874  u64_stats_update_end(&rx_ring->syncp);
1875  q_vector->rx.total_packets += total_rx_packets;
1876  q_vector->rx.total_bytes += total_rx_bytes;
1877 
1878  if (cleaned_count)
1879  ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1880 
1881  return !!budget;
1882 }
1883 
1891 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1892 {
1893  struct ixgbe_q_vector *q_vector;
1894  int v_idx;
1895  u32 mask;
1896 
1897  /* Populate MSIX to EITR Select */
1898  if (adapter->num_vfs > 32) {
1899  u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
1900  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
1901  }
1902 
1903  /*
1904  * Populate the IVAR table and set the ITR values to the
1905  * corresponding register.
1906  */
1907  for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1908  struct ixgbe_ring *ring;
1909  q_vector = adapter->q_vector[v_idx];
1910 
1911  ixgbe_for_each_ring(ring, q_vector->rx)
1912  ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1913 
1914  ixgbe_for_each_ring(ring, q_vector->tx)
1915  ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1916 
1917  if (q_vector->tx.ring && !q_vector->rx.ring) {
1918  /* tx only vector */
1919  if (adapter->tx_itr_setting == 1)
1920  q_vector->itr = IXGBE_10K_ITR;
1921  else
1922  q_vector->itr = adapter->tx_itr_setting;
1923  } else {
1924  /* rx or rx/tx vector */
1925  if (adapter->rx_itr_setting == 1)
1926  q_vector->itr = IXGBE_20K_ITR;
1927  else
1928  q_vector->itr = adapter->rx_itr_setting;
1929  }
1930 
1931  ixgbe_write_eitr(q_vector);
1932  }
1933 
1934  switch (adapter->hw.mac.type) {
1935  case ixgbe_mac_82598EB:
1936  ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1937  v_idx);
1938  break;
1939  case ixgbe_mac_82599EB:
1940  case ixgbe_mac_X540:
1941  ixgbe_set_ivar(adapter, -1, 1, v_idx);
1942  break;
1943  default:
1944  break;
1945  }
1946  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
1947 
1948  /* set up to autoclear timer, and the vectors */
1949  mask = IXGBE_EIMS_ENABLE_MASK;
1950  mask &= ~(IXGBE_EIMS_OTHER |
1952  IXGBE_EIMS_LSC);
1953 
1954  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
1955 }
1956 
1962 };
1963 
1979 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
1980  struct ixgbe_ring_container *ring_container)
1981 {
1982  int bytes = ring_container->total_bytes;
1983  int packets = ring_container->total_packets;
1984  u32 timepassed_us;
1985  u64 bytes_perint;
1986  u8 itr_setting = ring_container->itr;
1987 
1988  if (packets == 0)
1989  return;
1990 
1991  /* simple throttlerate management
1992  * 0-10MB/s lowest (100000 ints/s)
1993  * 10-20MB/s low (20000 ints/s)
1994  * 20-1249MB/s bulk (8000 ints/s)
1995  */
1996  /* what was last interrupt timeslice? */
1997  timepassed_us = q_vector->itr >> 2;
1998  bytes_perint = bytes / timepassed_us; /* bytes/usec */
1999 
2000  switch (itr_setting) {
2001  case lowest_latency:
2002  if (bytes_perint > 10)
2003  itr_setting = low_latency;
2004  break;
2005  case low_latency:
2006  if (bytes_perint > 20)
2007  itr_setting = bulk_latency;
2008  else if (bytes_perint <= 10)
2009  itr_setting = lowest_latency;
2010  break;
2011  case bulk_latency:
2012  if (bytes_perint <= 20)
2013  itr_setting = low_latency;
2014  break;
2015  }
2016 
2017  /* clear work counters since we have the values we need */
2018  ring_container->total_bytes = 0;
2019  ring_container->total_packets = 0;
2020 
2021  /* write updated itr to ring container */
2022  ring_container->itr = itr_setting;
2023 }
2024 
2033 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
2034 {
2035  struct ixgbe_adapter *adapter = q_vector->adapter;
2036  struct ixgbe_hw *hw = &adapter->hw;
2037  int v_idx = q_vector->v_idx;
2038  u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
2039 
2040  switch (adapter->hw.mac.type) {
2041  case ixgbe_mac_82598EB:
2042  /* must write high and low 16 bits to reset counter */
2043  itr_reg |= (itr_reg << 16);
2044  break;
2045  case ixgbe_mac_82599EB:
2046  case ixgbe_mac_X540:
2047  /*
2048  * set the WDIS bit to not clear the timer bits and cause an
2049  * immediate assertion of the interrupt
2050  */
2051  itr_reg |= IXGBE_EITR_CNT_WDIS;
2052  break;
2053  default:
2054  break;
2055  }
2056  IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
2057 }
2058 
2059 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
2060 {
2061  u32 new_itr = q_vector->itr;
2062  u8 current_itr;
2063 
2064  ixgbe_update_itr(q_vector, &q_vector->tx);
2065  ixgbe_update_itr(q_vector, &q_vector->rx);
2066 
2067  current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
2068 
2069  switch (current_itr) {
2070  /* counts and packets in update_itr are dependent on these numbers */
2071  case lowest_latency:
2072  new_itr = IXGBE_100K_ITR;
2073  break;
2074  case low_latency:
2075  new_itr = IXGBE_20K_ITR;
2076  break;
2077  case bulk_latency:
2078  new_itr = IXGBE_8K_ITR;
2079  break;
2080  default:
2081  break;
2082  }
2083 
2084  if (new_itr != q_vector->itr) {
2085  /* do an exponential smoothing */
2086  new_itr = (10 * new_itr * q_vector->itr) /
2087  ((9 * new_itr) + q_vector->itr);
2088 
2089  /* save the algorithm value here */
2090  q_vector->itr = new_itr;
2091 
2092  ixgbe_write_eitr(q_vector);
2093  }
2094 }
2095 
2100 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
2101 {
2102  struct ixgbe_hw *hw = &adapter->hw;
2103  u32 eicr = adapter->interrupt_event;
2104 
2105  if (test_bit(__IXGBE_DOWN, &adapter->state))
2106  return;
2107 
2108  if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
2109  !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
2110  return;
2111 
2113 
2114  switch (hw->device_id) {
2116  /*
2117  * Since the warning interrupt is for both ports
2118  * we don't have to check if:
2119  * - This interrupt wasn't for our port.
2120  * - We may have missed the interrupt so always have to
2121  * check if we got a LSC
2122  */
2123  if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
2124  !(eicr & IXGBE_EICR_LSC))
2125  return;
2126 
2127  if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
2128  u32 autoneg;
2129  bool link_up = false;
2130 
2131  hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
2132 
2133  if (link_up)
2134  return;
2135  }
2136 
2137  /* Check if this is not due to overtemp */
2138  if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
2139  return;
2140 
2141  break;
2142  default:
2143  if (!(eicr & IXGBE_EICR_GPI_SDP0))
2144  return;
2145  break;
2146  }
2147  e_crit(drv,
2148  "Network adapter has been stopped because it has over heated. "
2149  "Restart the computer. If the problem persists, "
2150  "power off the system and replace the adapter\n");
2151 
2152  adapter->interrupt_event = 0;
2153 }
2154 
2155 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
2156 {
2157  struct ixgbe_hw *hw = &adapter->hw;
2158 
2159  if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
2160  (eicr & IXGBE_EICR_GPI_SDP1)) {
2161  e_crit(probe, "Fan has stopped, replace the adapter\n");
2162  /* write to clear the interrupt */
2163  IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
2164  }
2165 }
2166 
2167 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
2168 {
2169  if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
2170  return;
2171 
2172  switch (adapter->hw.mac.type) {
2173  case ixgbe_mac_82599EB:
2174  /*
2175  * Need to check link state so complete overtemp check
2176  * on service task
2177  */
2178  if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
2179  (!test_bit(__IXGBE_DOWN, &adapter->state))) {
2180  adapter->interrupt_event = eicr;
2182  ixgbe_service_event_schedule(adapter);
2183  return;
2184  }
2185  return;
2186  case ixgbe_mac_X540:
2187  if (!(eicr & IXGBE_EICR_TS))
2188  return;
2189  break;
2190  default:
2191  return;
2192  }
2193 
2194  e_crit(drv,
2195  "Network adapter has been stopped because it has over heated. "
2196  "Restart the computer. If the problem persists, "
2197  "power off the system and replace the adapter\n");
2198 }
2199 
2200 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
2201 {
2202  struct ixgbe_hw *hw = &adapter->hw;
2203 
2204  if (eicr & IXGBE_EICR_GPI_SDP2) {
2205  /* Clear the interrupt */
2206  IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
2207  if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2208  adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
2209  ixgbe_service_event_schedule(adapter);
2210  }
2211  }
2212 
2213  if (eicr & IXGBE_EICR_GPI_SDP1) {
2214  /* Clear the interrupt */
2215  IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
2216  if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2217  adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
2218  ixgbe_service_event_schedule(adapter);
2219  }
2220  }
2221 }
2222 
2223 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
2224 {
2225  struct ixgbe_hw *hw = &adapter->hw;
2226 
2227  adapter->lsc_int++;
2228  adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2229  adapter->link_check_timeout = jiffies;
2230  if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2232  IXGBE_WRITE_FLUSH(hw);
2233  ixgbe_service_event_schedule(adapter);
2234  }
2235 }
2236 
2237 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
2238  u64 qmask)
2239 {
2240  u32 mask;
2241  struct ixgbe_hw *hw = &adapter->hw;
2242 
2243  switch (hw->mac.type) {
2244  case ixgbe_mac_82598EB:
2245  mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
2246  IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
2247  break;
2248  case ixgbe_mac_82599EB:
2249  case ixgbe_mac_X540:
2250  mask = (qmask & 0xFFFFFFFF);
2251  if (mask)
2252  IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
2253  mask = (qmask >> 32);
2254  if (mask)
2255  IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
2256  break;
2257  default:
2258  break;
2259  }
2260  /* skip the flush */
2261 }
2262 
2263 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
2264  u64 qmask)
2265 {
2266  u32 mask;
2267  struct ixgbe_hw *hw = &adapter->hw;
2268 
2269  switch (hw->mac.type) {
2270  case ixgbe_mac_82598EB:
2271  mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
2272  IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
2273  break;
2274  case ixgbe_mac_82599EB:
2275  case ixgbe_mac_X540:
2276  mask = (qmask & 0xFFFFFFFF);
2277  if (mask)
2278  IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
2279  mask = (qmask >> 32);
2280  if (mask)
2281  IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
2282  break;
2283  default:
2284  break;
2285  }
2286  /* skip the flush */
2287 }
2288 
2293 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
2294  bool flush)
2295 {
2297 
2298  /* don't reenable LSC while waiting for link */
2299  if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
2300  mask &= ~IXGBE_EIMS_LSC;
2301 
2302  if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
2303  switch (adapter->hw.mac.type) {
2304  case ixgbe_mac_82599EB:
2305  mask |= IXGBE_EIMS_GPI_SDP0;
2306  break;
2307  case ixgbe_mac_X540:
2308  mask |= IXGBE_EIMS_TS;
2309  break;
2310  default:
2311  break;
2312  }
2313  if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
2314  mask |= IXGBE_EIMS_GPI_SDP1;
2315  switch (adapter->hw.mac.type) {
2316  case ixgbe_mac_82599EB:
2317  mask |= IXGBE_EIMS_GPI_SDP1;
2318  mask |= IXGBE_EIMS_GPI_SDP2;
2319  case ixgbe_mac_X540:
2320  mask |= IXGBE_EIMS_ECC;
2321  mask |= IXGBE_EIMS_MAILBOX;
2322  break;
2323  default:
2324  break;
2325  }
2326 
2327 #ifdef CONFIG_IXGBE_PTP
2328  if (adapter->hw.mac.type == ixgbe_mac_X540)
2329  mask |= IXGBE_EIMS_TIMESYNC;
2330 #endif
2331 
2332  if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2334  mask |= IXGBE_EIMS_FLOW_DIR;
2335 
2336  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2337  if (queues)
2338  ixgbe_irq_enable_queues(adapter, ~0);
2339  if (flush)
2340  IXGBE_WRITE_FLUSH(&adapter->hw);
2341 }
2342 
2343 static irqreturn_t ixgbe_msix_other(int irq, void *data)
2344 {
2345  struct ixgbe_adapter *adapter = data;
2346  struct ixgbe_hw *hw = &adapter->hw;
2347  u32 eicr;
2348 
2349  /*
2350  * Workaround for Silicon errata. Use clear-by-write instead
2351  * of clear-by-read. Reading with EICS will return the
2352  * interrupt causes without clearing, which later be done
2353  * with the write to EICR.
2354  */
2355  eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
2356  IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
2357 
2358  if (eicr & IXGBE_EICR_LSC)
2359  ixgbe_check_lsc(adapter);
2360 
2361  if (eicr & IXGBE_EICR_MAILBOX)
2362  ixgbe_msg_task(adapter);
2363 
2364  switch (hw->mac.type) {
2365  case ixgbe_mac_82599EB:
2366  case ixgbe_mac_X540:
2367  if (eicr & IXGBE_EICR_ECC)
2368  e_info(link, "Received unrecoverable ECC Err, please "
2369  "reboot\n");
2370  /* Handle Flow Director Full threshold interrupt */
2371  if (eicr & IXGBE_EICR_FLOW_DIR) {
2372  int reinit_count = 0;
2373  int i;
2374  for (i = 0; i < adapter->num_tx_queues; i++) {
2375  struct ixgbe_ring *ring = adapter->tx_ring[i];
2377  &ring->state))
2378  reinit_count++;
2379  }
2380  if (reinit_count) {
2381  /* no more flow director interrupts until after init */
2384  ixgbe_service_event_schedule(adapter);
2385  }
2386  }
2387  ixgbe_check_sfp_event(adapter, eicr);
2388  ixgbe_check_overtemp_event(adapter, eicr);
2389  break;
2390  default:
2391  break;
2392  }
2393 
2394  ixgbe_check_fan_failure(adapter, eicr);
2395 
2396 #ifdef CONFIG_IXGBE_PTP
2397  if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
2398  ixgbe_ptp_check_pps_event(adapter, eicr);
2399 #endif
2400 
2401  /* re-enable the original interrupt state, no lsc, no queues */
2402  if (!test_bit(__IXGBE_DOWN, &adapter->state))
2403  ixgbe_irq_enable(adapter, false, false);
2404 
2405  return IRQ_HANDLED;
2406 }
2407 
2408 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
2409 {
2410  struct ixgbe_q_vector *q_vector = data;
2411 
2412  /* EIAM disabled interrupts (on this vector) for us */
2413 
2414  if (q_vector->rx.ring || q_vector->tx.ring)
2415  napi_schedule(&q_vector->napi);
2416 
2417  return IRQ_HANDLED;
2418 }
2419 
2427 int ixgbe_poll(struct napi_struct *napi, int budget)
2428 {
2429  struct ixgbe_q_vector *q_vector =
2430  container_of(napi, struct ixgbe_q_vector, napi);
2431  struct ixgbe_adapter *adapter = q_vector->adapter;
2432  struct ixgbe_ring *ring;
2433  int per_ring_budget;
2434  bool clean_complete = true;
2435 
2436 #ifdef CONFIG_IXGBE_DCA
2437  if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2438  ixgbe_update_dca(q_vector);
2439 #endif
2440 
2441  ixgbe_for_each_ring(ring, q_vector->tx)
2442  clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring);
2443 
2444  /* attempt to distribute budget to each queue fairly, but don't allow
2445  * the budget to go below 1 because we'll exit polling */
2446  if (q_vector->rx.count > 1)
2447  per_ring_budget = max(budget/q_vector->rx.count, 1);
2448  else
2449  per_ring_budget = budget;
2450 
2451  ixgbe_for_each_ring(ring, q_vector->rx)
2452  clean_complete &= ixgbe_clean_rx_irq(q_vector, ring,
2453  per_ring_budget);
2454 
2455  /* If all work not completed, return budget and keep polling */
2456  if (!clean_complete)
2457  return budget;
2458 
2459  /* all work done, exit the polling mode */
2460  napi_complete(napi);
2461  if (adapter->rx_itr_setting & 1)
2462  ixgbe_set_itr(q_vector);
2463  if (!test_bit(__IXGBE_DOWN, &adapter->state))
2464  ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx));
2465 
2466  return 0;
2467 }
2468 
2476 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2477 {
2478  struct net_device *netdev = adapter->netdev;
2479  int vector, err;
2480  int ri = 0, ti = 0;
2481 
2482  for (vector = 0; vector < adapter->num_q_vectors; vector++) {
2483  struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2484  struct msix_entry *entry = &adapter->msix_entries[vector];
2485 
2486  if (q_vector->tx.ring && q_vector->rx.ring) {
2487  snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2488  "%s-%s-%d", netdev->name, "TxRx", ri++);
2489  ti++;
2490  } else if (q_vector->rx.ring) {
2491  snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2492  "%s-%s-%d", netdev->name, "rx", ri++);
2493  } else if (q_vector->tx.ring) {
2494  snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2495  "%s-%s-%d", netdev->name, "tx", ti++);
2496  } else {
2497  /* skip this unused q_vector */
2498  continue;
2499  }
2500  err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2501  q_vector->name, q_vector);
2502  if (err) {
2503  e_err(probe, "request_irq failed for MSIX interrupt "
2504  "Error: %d\n", err);
2505  goto free_queue_irqs;
2506  }
2507  /* If Flow Director is enabled, set interrupt affinity */
2508  if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2509  /* assign the mask for this irq */
2510  irq_set_affinity_hint(entry->vector,
2511  &q_vector->affinity_mask);
2512  }
2513  }
2514 
2515  err = request_irq(adapter->msix_entries[vector].vector,
2516  ixgbe_msix_other, 0, netdev->name, adapter);
2517  if (err) {
2518  e_err(probe, "request_irq for msix_other failed: %d\n", err);
2519  goto free_queue_irqs;
2520  }
2521 
2522  return 0;
2523 
2524 free_queue_irqs:
2525  while (vector) {
2526  vector--;
2527  irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2528  NULL);
2529  free_irq(adapter->msix_entries[vector].vector,
2530  adapter->q_vector[vector]);
2531  }
2532  adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2533  pci_disable_msix(adapter->pdev);
2534  kfree(adapter->msix_entries);
2535  adapter->msix_entries = NULL;
2536  return err;
2537 }
2538 
2544 static irqreturn_t ixgbe_intr(int irq, void *data)
2545 {
2546  struct ixgbe_adapter *adapter = data;
2547  struct ixgbe_hw *hw = &adapter->hw;
2548  struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2549  u32 eicr;
2550 
2551  /*
2552  * Workaround for silicon errata #26 on 82598. Mask the interrupt
2553  * before the read of EICR.
2554  */
2556 
2557  /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2558  * therefore no explicit interrupt disable is necessary */
2559  eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2560  if (!eicr) {
2561  /*
2562  * shared interrupt alert!
2563  * make sure interrupts are enabled because the read will
2564  * have disabled interrupts due to EIAM
2565  * finish the workaround of silicon errata on 82598. Unmask
2566  * the interrupt that we masked before the EICR read.
2567  */
2568  if (!test_bit(__IXGBE_DOWN, &adapter->state))
2569  ixgbe_irq_enable(adapter, true, true);
2570  return IRQ_NONE; /* Not our interrupt */
2571  }
2572 
2573  if (eicr & IXGBE_EICR_LSC)
2574  ixgbe_check_lsc(adapter);
2575 
2576  switch (hw->mac.type) {
2577  case ixgbe_mac_82599EB:
2578  ixgbe_check_sfp_event(adapter, eicr);
2579  /* Fall through */
2580  case ixgbe_mac_X540:
2581  if (eicr & IXGBE_EICR_ECC)
2582  e_info(link, "Received unrecoverable ECC err, please "
2583  "reboot\n");
2584  ixgbe_check_overtemp_event(adapter, eicr);
2585  break;
2586  default:
2587  break;
2588  }
2589 
2590  ixgbe_check_fan_failure(adapter, eicr);
2591 #ifdef CONFIG_IXGBE_PTP
2592  if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
2593  ixgbe_ptp_check_pps_event(adapter, eicr);
2594 #endif
2595 
2596  /* would disable interrupts here but EIAM disabled it */
2597  napi_schedule(&q_vector->napi);
2598 
2599  /*
2600  * re-enable link(maybe) and non-queue interrupts, no flush.
2601  * ixgbe_poll will re-enable the queue interrupts
2602  */
2603  if (!test_bit(__IXGBE_DOWN, &adapter->state))
2604  ixgbe_irq_enable(adapter, false, false);
2605 
2606  return IRQ_HANDLED;
2607 }
2608 
2616 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2617 {
2618  struct net_device *netdev = adapter->netdev;
2619  int err;
2620 
2621  if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
2622  err = ixgbe_request_msix_irqs(adapter);
2623  else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
2624  err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2625  netdev->name, adapter);
2626  else
2627  err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2628  netdev->name, adapter);
2629 
2630  if (err)
2631  e_err(probe, "request_irq failed, Error %d\n", err);
2632 
2633  return err;
2634 }
2635 
2636 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2637 {
2638  int vector;
2639 
2640  if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
2641  free_irq(adapter->pdev->irq, adapter);
2642  return;
2643  }
2644 
2645  for (vector = 0; vector < adapter->num_q_vectors; vector++) {
2646  struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2647  struct msix_entry *entry = &adapter->msix_entries[vector];
2648 
2649  /* free only the irqs that were actually requested */
2650  if (!q_vector->rx.ring && !q_vector->tx.ring)
2651  continue;
2652 
2653  /* clear the affinity_mask in the IRQ descriptor */
2654  irq_set_affinity_hint(entry->vector, NULL);
2655 
2656  free_irq(entry->vector, q_vector);
2657  }
2658 
2659  free_irq(adapter->msix_entries[vector++].vector, adapter);
2660 }
2661 
2666 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2667 {
2668  switch (adapter->hw.mac.type) {
2669  case ixgbe_mac_82598EB:
2670  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2671  break;
2672  case ixgbe_mac_82599EB:
2673  case ixgbe_mac_X540:
2674  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2675  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2676  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2677  break;
2678  default:
2679  break;
2680  }
2681  IXGBE_WRITE_FLUSH(&adapter->hw);
2682  if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2683  int vector;
2684 
2685  for (vector = 0; vector < adapter->num_q_vectors; vector++)
2686  synchronize_irq(adapter->msix_entries[vector].vector);
2687 
2688  synchronize_irq(adapter->msix_entries[vector++].vector);
2689  } else {
2690  synchronize_irq(adapter->pdev->irq);
2691  }
2692 }
2693 
2698 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2699 {
2700  struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2701 
2702  /* rx/tx vector */
2703  if (adapter->rx_itr_setting == 1)
2704  q_vector->itr = IXGBE_20K_ITR;
2705  else
2706  q_vector->itr = adapter->rx_itr_setting;
2707 
2708  ixgbe_write_eitr(q_vector);
2709 
2710  ixgbe_set_ivar(adapter, 0, 0, 0);
2711  ixgbe_set_ivar(adapter, 1, 0, 0);
2712 
2713  e_info(hw, "Legacy interrupt IVAR setup done\n");
2714 }
2715 
2724  struct ixgbe_ring *ring)
2725 {
2726  struct ixgbe_hw *hw = &adapter->hw;
2727  u64 tdba = ring->dma;
2728  int wait_loop = 10;
2729  u32 txdctl = IXGBE_TXDCTL_ENABLE;
2730  u8 reg_idx = ring->reg_idx;
2731 
2732  /* disable queue to avoid issues while updating state */
2733  IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
2734  IXGBE_WRITE_FLUSH(hw);
2735 
2736  IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2737  (tdba & DMA_BIT_MASK(32)));
2738  IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2739  IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2740  ring->count * sizeof(union ixgbe_adv_tx_desc));
2741  IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2742  IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2743  ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2744 
2745  /*
2746  * set WTHRESH to encourage burst writeback, it should not be set
2747  * higher than 1 when ITR is 0 as it could cause false TX hangs
2748  *
2749  * In order to avoid issues WTHRESH + PTHRESH should always be equal
2750  * to or less than the number of on chip descriptors, which is
2751  * currently 40.
2752  */
2753  if (!ring->q_vector || (ring->q_vector->itr < 8))
2754  txdctl |= (1 << 16); /* WTHRESH = 1 */
2755  else
2756  txdctl |= (8 << 16); /* WTHRESH = 8 */
2757 
2758  /*
2759  * Setting PTHRESH to 32 both improves performance
2760  * and avoids a TX hang with DFP enabled
2761  */
2762  txdctl |= (1 << 8) | /* HTHRESH = 1 */
2763  32; /* PTHRESH = 32 */
2764 
2765  /* reinitialize flowdirector state */
2766  if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2767  ring->atr_sample_rate = adapter->atr_sample_rate;
2768  ring->atr_count = 0;
2770  } else {
2771  ring->atr_sample_rate = 0;
2772  }
2773 
2775 
2776  /* enable queue */
2777  IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2778 
2779  /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2780  if (hw->mac.type == ixgbe_mac_82598EB &&
2782  return;
2783 
2784  /* poll to verify queue is enabled */
2785  do {
2786  usleep_range(1000, 2000);
2787  txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2788  } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2789  if (!wait_loop)
2790  e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2791 }
2792 
2793 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2794 {
2795  struct ixgbe_hw *hw = &adapter->hw;
2796  u32 rttdcs, mtqc;
2797  u8 tcs = netdev_get_num_tc(adapter->netdev);
2798 
2799  if (hw->mac.type == ixgbe_mac_82598EB)
2800  return;
2801 
2802  /* disable the arbiter while setting MTQC */
2803  rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2804  rttdcs |= IXGBE_RTTDCS_ARBDIS;
2805  IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2806 
2807  /* set transmit pool layout */
2808  if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2809  mtqc = IXGBE_MTQC_VT_ENA;
2810  if (tcs > 4)
2812  else if (tcs > 1)
2814  else if (adapter->ring_feature[RING_F_RSS].indices == 4)
2815  mtqc |= IXGBE_MTQC_32VF;
2816  else
2817  mtqc |= IXGBE_MTQC_64VF;
2818  } else {
2819  if (tcs > 4)
2821  else if (tcs > 1)
2823  else
2824  mtqc = IXGBE_MTQC_64Q_1PB;
2825  }
2826 
2827  IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
2828 
2829  /* Enable Security TX Buffer IFG for multiple pb */
2830  if (tcs) {
2831  u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2832  sectx |= IXGBE_SECTX_DCB;
2833  IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
2834  }
2835 
2836  /* re-enable the arbiter */
2837  rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2838  IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2839 }
2840 
2847 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2848 {
2849  struct ixgbe_hw *hw = &adapter->hw;
2850  u32 dmatxctl;
2851  u32 i;
2852 
2853  ixgbe_setup_mtqc(adapter);
2854 
2855  if (hw->mac.type != ixgbe_mac_82598EB) {
2856  /* DMATXCTL.EN must be before Tx queues are enabled */
2857  dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2858  dmatxctl |= IXGBE_DMATXCTL_TE;
2859  IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2860  }
2861 
2862  /* Setup the HW Tx Head and Tail descriptor pointers */
2863  for (i = 0; i < adapter->num_tx_queues; i++)
2864  ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2865 }
2866 
2867 static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
2868  struct ixgbe_ring *ring)
2869 {
2870  struct ixgbe_hw *hw = &adapter->hw;
2871  u8 reg_idx = ring->reg_idx;
2872  u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
2873 
2874  srrctl |= IXGBE_SRRCTL_DROP_EN;
2875 
2876  IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
2877 }
2878 
2879 static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
2880  struct ixgbe_ring *ring)
2881 {
2882  struct ixgbe_hw *hw = &adapter->hw;
2883  u8 reg_idx = ring->reg_idx;
2884  u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
2885 
2886  srrctl &= ~IXGBE_SRRCTL_DROP_EN;
2887 
2888  IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
2889 }
2890 
2891 #ifdef CONFIG_IXGBE_DCB
2892 void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
2893 #else
2894 static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
2895 #endif
2896 {
2897  int i;
2898  bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
2899 
2900  if (adapter->ixgbe_ieee_pfc)
2901  pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
2902 
2903  /*
2904  * We should set the drop enable bit if:
2905  * SR-IOV is enabled
2906  * or
2907  * Number of Rx queues > 1 and flow control is disabled
2908  *
2909  * This allows us to avoid head of line blocking for security
2910  * and performance reasons.
2911  */
2912  if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
2913  !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
2914  for (i = 0; i < adapter->num_rx_queues; i++)
2915  ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
2916  } else {
2917  for (i = 0; i < adapter->num_rx_queues; i++)
2918  ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
2919  }
2920 }
2921 
2922 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2923 
2924 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2925  struct ixgbe_ring *rx_ring)
2926 {
2927  struct ixgbe_hw *hw = &adapter->hw;
2928  u32 srrctl;
2929  u8 reg_idx = rx_ring->reg_idx;
2930 
2931  if (hw->mac.type == ixgbe_mac_82598EB) {
2932  u16 mask = adapter->ring_feature[RING_F_RSS].mask;
2933 
2934  /*
2935  * if VMDq is not active we must program one srrctl register
2936  * per RSS queue since we have enabled RDRXCTL.MVMEN
2937  */
2938  reg_idx &= mask;
2939  }
2940 
2941  /* configure header buffer length, needed for RSC */
2943 
2944  /* configure the packet buffer length */
2945  srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2946 
2947  /* configure descriptor type */
2949 
2950  IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
2951 }
2952 
2953 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2954 {
2955  struct ixgbe_hw *hw = &adapter->hw;
2956  static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
2957  0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
2958  0x6A3E67EA, 0x14364D17, 0x3BED200D};
2959  u32 mrqc = 0, reta = 0;
2960  u32 rxcsum;
2961  int i, j;
2962  u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
2963 
2964  /*
2965  * Program table for at least 2 queues w/ SR-IOV so that VFs can
2966  * make full use of any rings they may have. We will use the
2967  * PSRTYPE register to control how many rings we use within the PF.
2968  */
2969  if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2))
2970  rss_i = 2;
2971 
2972  /* Fill out hash function seeds */
2973  for (i = 0; i < 10; i++)
2974  IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
2975 
2976  /* Fill out redirection table */
2977  for (i = 0, j = 0; i < 128; i++, j++) {
2978  if (j == rss_i)
2979  j = 0;
2980  /* reta = 4-byte sliding window of
2981  * 0x00..(indices-1)(indices-1)00..etc. */
2982  reta = (reta << 8) | (j * 0x11);
2983  if ((i & 3) == 3)
2984  IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2985  }
2986 
2987  /* Disable indicating checksum in descriptor, enables RSS hash */
2988  rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2989  rxcsum |= IXGBE_RXCSUM_PCSD;
2990  IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2991 
2992  if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
2993  if (adapter->ring_feature[RING_F_RSS].mask)
2994  mrqc = IXGBE_MRQC_RSSEN;
2995  } else {
2996  u8 tcs = netdev_get_num_tc(adapter->netdev);
2997 
2998  if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2999  if (tcs > 4)
3000  mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */
3001  else if (tcs > 1)
3002  mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */
3003  else if (adapter->ring_feature[RING_F_RSS].indices == 4)
3004  mrqc = IXGBE_MRQC_VMDQRSS32EN;
3005  else
3006  mrqc = IXGBE_MRQC_VMDQRSS64EN;
3007  } else {
3008  if (tcs > 4)
3009  mrqc = IXGBE_MRQC_RTRSS8TCEN;
3010  else if (tcs > 1)
3011  mrqc = IXGBE_MRQC_RTRSS4TCEN;
3012  else
3013  mrqc = IXGBE_MRQC_RSSEN;
3014  }
3015  }
3016 
3017  /* Perform hash on these packet types */
3018  mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4 |
3022 
3023  if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
3025  if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
3027 
3028  IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3029 }
3030 
3036 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
3037  struct ixgbe_ring *ring)
3038 {
3039  struct ixgbe_hw *hw = &adapter->hw;
3040  u32 rscctrl;
3041  u8 reg_idx = ring->reg_idx;
3042 
3043  if (!ring_is_rsc_enabled(ring))
3044  return;
3045 
3046  rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
3047  rscctrl |= IXGBE_RSCCTL_RSCEN;
3048  /*
3049  * we must limit the number of descriptors so that the
3050  * total size of max desc * buf_len is not greater
3051  * than 65536
3052  */
3053  rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3054  IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
3055 }
3056 
3057 #define IXGBE_MAX_RX_DESC_POLL 10
3058 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
3059  struct ixgbe_ring *ring)
3060 {
3061  struct ixgbe_hw *hw = &adapter->hw;
3062  int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3063  u32 rxdctl;
3064  u8 reg_idx = ring->reg_idx;
3065 
3066  /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3067  if (hw->mac.type == ixgbe_mac_82598EB &&
3069  return;
3070 
3071  do {
3072  usleep_range(1000, 2000);
3073  rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3074  } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
3075 
3076  if (!wait_loop) {
3077  e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
3078  "the polling period\n", reg_idx);
3079  }
3080 }
3081 
3083  struct ixgbe_ring *ring)
3084 {
3085  struct ixgbe_hw *hw = &adapter->hw;
3086  int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3087  u32 rxdctl;
3088  u8 reg_idx = ring->reg_idx;
3089 
3090  rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3091  rxdctl &= ~IXGBE_RXDCTL_ENABLE;
3092 
3093  /* write value back with RXDCTL.ENABLE bit cleared */
3094  IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3095 
3096  if (hw->mac.type == ixgbe_mac_82598EB &&
3098  return;
3099 
3100  /* the hardware may take up to 100us to really disable the rx queue */
3101  do {
3102  udelay(10);
3103  rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3104  } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
3105 
3106  if (!wait_loop) {
3107  e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
3108  "the polling period\n", reg_idx);
3109  }
3110 }
3111 
3113  struct ixgbe_ring *ring)
3114 {
3115  struct ixgbe_hw *hw = &adapter->hw;
3116  u64 rdba = ring->dma;
3117  u32 rxdctl;
3118  u8 reg_idx = ring->reg_idx;
3119 
3120  /* disable queue to avoid issues while updating state */
3121  rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3122  ixgbe_disable_rx_queue(adapter, ring);
3123 
3124  IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
3125  IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
3126  IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
3127  ring->count * sizeof(union ixgbe_adv_rx_desc));
3128  IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
3129  IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
3130  ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
3131 
3132  ixgbe_configure_srrctl(adapter, ring);
3133  ixgbe_configure_rscctl(adapter, ring);
3134 
3135  /* If operating in IOV mode set RLPML for X540 */
3136  if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
3137  hw->mac.type == ixgbe_mac_X540) {
3138  rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
3139  rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
3141  }
3142 
3143  if (hw->mac.type == ixgbe_mac_82598EB) {
3144  /*
3145  * enable cache line friendly hardware writes:
3146  * PTHRESH=32 descriptors (half the internal cache),
3147  * this also removes ugly rx_no_buffer_count increment
3148  * HTHRESH=4 descriptors (to minimize latency on fetch)
3149  * WTHRESH=8 burst writeback up to two cache lines
3150  */
3151  rxdctl &= ~0x3FFFFF;
3152  rxdctl |= 0x080420;
3153  }
3154 
3155  /* enable receive descriptor ring */
3156  rxdctl |= IXGBE_RXDCTL_ENABLE;
3157  IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3158 
3159  ixgbe_rx_desc_queue_enable(adapter, ring);
3160  ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
3161 }
3162 
3163 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
3164 {
3165  struct ixgbe_hw *hw = &adapter->hw;
3166  int rss_i = adapter->ring_feature[RING_F_RSS].indices;
3167  int p;
3168 
3169  /* PSRTYPE must be initialized in non 82598 adapters */
3170  u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
3175 
3176  if (hw->mac.type == ixgbe_mac_82598EB)
3177  return;
3178 
3179  if (rss_i > 3)
3180  psrtype |= 2 << 29;
3181  else if (rss_i > 1)
3182  psrtype |= 1 << 29;
3183 
3184  for (p = 0; p < adapter->num_rx_pools; p++)
3186  psrtype);
3187 }
3188 
3189 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
3190 {
3191  struct ixgbe_hw *hw = &adapter->hw;
3192  u32 reg_offset, vf_shift;
3193  u32 gcr_ext, vmdctl;
3194  int i;
3195 
3196  if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
3197  return;
3198 
3199  vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3200  vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;
3201  vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
3202  vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;
3203  vmdctl |= IXGBE_VT_CTL_REPLEN;
3204  IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
3205 
3206  vf_shift = VMDQ_P(0) % 32;
3207  reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;
3208 
3209  /* Enable only the PF's pool for Tx/Rx */
3210  IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (~0) << vf_shift);
3211  IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);
3212  IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (~0) << vf_shift);
3213  IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);
3215 
3216  /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
3217  hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));
3218 
3219  /*
3220  * Set up VF register offsets for selected VT Mode,
3221  * i.e. 32 or 64 VFs for SR-IOV
3222  */
3223  switch (adapter->ring_feature[RING_F_VMDQ].mask) {
3225  gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;
3226  break;
3228  gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;
3229  break;
3230  default:
3231  gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;
3232  break;
3233  }
3234 
3235  IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
3236 
3237  /* enable Tx loopback for VF/PF communication */
3239 
3240  /* Enable MAC Anti-Spoofing */
3241  hw->mac.ops.set_mac_anti_spoofing(hw, (adapter->num_vfs != 0),
3242  adapter->num_vfs);
3243  /* For VFs that have spoof checking turned off */
3244  for (i = 0; i < adapter->num_vfs; i++) {
3245  if (!adapter->vfinfo[i].spoofchk_enabled)
3246  ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);
3247  }
3248 }
3249 
3250 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
3251 {
3252  struct ixgbe_hw *hw = &adapter->hw;
3253  struct net_device *netdev = adapter->netdev;
3254  int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3255  struct ixgbe_ring *rx_ring;
3256  int i;
3257  u32 mhadd, hlreg0;
3258 
3259 #ifdef IXGBE_FCOE
3260  /* adjust max frame to be able to do baby jumbo for FCoE */
3261  if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
3262  (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
3263  max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
3264 
3265 #endif /* IXGBE_FCOE */
3266  mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3267  if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
3268  mhadd &= ~IXGBE_MHADD_MFS_MASK;
3269  mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
3270 
3271  IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3272  }
3273 
3274  /* MHADD will allow an extra 4 bytes past for vlan tagged frames */
3275  max_frame += VLAN_HLEN;
3276 
3277  hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3278  /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
3279  hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3280  IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3281 
3282  /*
3283  * Setup the HW Rx Head and Tail Descriptor Pointers and
3284  * the Base and Length of the Rx Descriptor Ring
3285  */
3286  for (i = 0; i < adapter->num_rx_queues; i++) {
3287  rx_ring = adapter->rx_ring[i];
3288  if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
3289  set_ring_rsc_enabled(rx_ring);
3290  else
3291  clear_ring_rsc_enabled(rx_ring);
3292  }
3293 }
3294 
3295 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
3296 {
3297  struct ixgbe_hw *hw = &adapter->hw;
3298  u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3299 
3300  switch (hw->mac.type) {
3301  case ixgbe_mac_82598EB:
3302  /*
3303  * For VMDq support of different descriptor types or
3304  * buffer sizes through the use of multiple SRRCTL
3305  * registers, RDRXCTL.MVMEN must be set to 1
3306  *
3307  * also, the manual doesn't mention it clearly but DCA hints
3308  * will only use queue 0's tags unless this bit is set. Side
3309  * effects of setting this bit are only that SRRCTL must be
3310  * fully programmed [0..15]
3311  */
3312  rdrxctl |= IXGBE_RDRXCTL_MVMEN;
3313  break;
3314  case ixgbe_mac_82599EB:
3315  case ixgbe_mac_X540:
3316  /* Disable RSC for ACK packets */
3319  rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3320  /* hardware requires some bits to be set by default */
3322  rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3323  break;
3324  default:
3325  /* We should do nothing since we don't know this hardware */
3326  return;
3327  }
3328 
3329  IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3330 }
3331 
3338 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
3339 {
3340  struct ixgbe_hw *hw = &adapter->hw;
3341  int i;
3342  u32 rxctrl;
3343 
3344  /* disable receives while setting up the descriptors */
3345  rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3347 
3348  ixgbe_setup_psrtype(adapter);
3349  ixgbe_setup_rdrxctl(adapter);
3350 
3351  /* Program registers for the distribution of queues */
3352  ixgbe_setup_mrqc(adapter);
3353 
3354  /* set_rx_buffer_len must be called before ring initialization */
3355  ixgbe_set_rx_buffer_len(adapter);
3356 
3357  /*
3358  * Setup the HW Rx Head and Tail Descriptor Pointers and
3359  * the Base and Length of the Rx Descriptor Ring
3360  */
3361  for (i = 0; i < adapter->num_rx_queues; i++)
3362  ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
3363 
3364  /* disable drop enable for 82598 parts */
3365  if (hw->mac.type == ixgbe_mac_82598EB)
3366  rxctrl |= IXGBE_RXCTRL_DMBYPS;
3367 
3368  /* enable all receives */
3369  rxctrl |= IXGBE_RXCTRL_RXEN;
3370  hw->mac.ops.enable_rx_dma(hw, rxctrl);
3371 }
3372 
3373 static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
3374 {
3375  struct ixgbe_adapter *adapter = netdev_priv(netdev);
3376  struct ixgbe_hw *hw = &adapter->hw;
3377 
3378  /* add VID to filter table */
3379  hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true);
3380  set_bit(vid, adapter->active_vlans);
3381 
3382  return 0;
3383 }
3384 
3385 static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
3386 {
3387  struct ixgbe_adapter *adapter = netdev_priv(netdev);
3388  struct ixgbe_hw *hw = &adapter->hw;
3389 
3390  /* remove VID from filter table */
3391  hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), false);
3392  clear_bit(vid, adapter->active_vlans);
3393 
3394  return 0;
3395 }
3396 
3401 static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
3402 {
3403  struct ixgbe_hw *hw = &adapter->hw;
3404  u32 vlnctrl;
3405 
3406  vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3407  vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
3408  IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3409 }
3410 
3415 static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
3416 {
3417  struct ixgbe_hw *hw = &adapter->hw;
3418  u32 vlnctrl;
3419 
3420  vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3421  vlnctrl |= IXGBE_VLNCTRL_VFE;
3422  vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
3423  IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3424 }
3425 
3430 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
3431 {
3432  struct ixgbe_hw *hw = &adapter->hw;
3433  u32 vlnctrl;
3434  int i, j;
3435 
3436  switch (hw->mac.type) {
3437  case ixgbe_mac_82598EB:
3438  vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3439  vlnctrl &= ~IXGBE_VLNCTRL_VME;
3440  IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3441  break;
3442  case ixgbe_mac_82599EB:
3443  case ixgbe_mac_X540:
3444  for (i = 0; i < adapter->num_rx_queues; i++) {
3445  j = adapter->rx_ring[i]->reg_idx;
3446  vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3447  vlnctrl &= ~IXGBE_RXDCTL_VME;
3448  IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3449  }
3450  break;
3451  default:
3452  break;
3453  }
3454 }
3455 
3460 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
3461 {
3462  struct ixgbe_hw *hw = &adapter->hw;
3463  u32 vlnctrl;
3464  int i, j;
3465 
3466  switch (hw->mac.type) {
3467  case ixgbe_mac_82598EB:
3468  vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3469  vlnctrl |= IXGBE_VLNCTRL_VME;
3470  IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3471  break;
3472  case ixgbe_mac_82599EB:
3473  case ixgbe_mac_X540:
3474  for (i = 0; i < adapter->num_rx_queues; i++) {
3475  j = adapter->rx_ring[i]->reg_idx;
3476  vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3477  vlnctrl |= IXGBE_RXDCTL_VME;
3478  IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3479  }
3480  break;
3481  default:
3482  break;
3483  }
3484 }
3485 
3486 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
3487 {
3488  u16 vid;
3489 
3490  ixgbe_vlan_rx_add_vid(adapter->netdev, 0);
3491 
3492  for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
3493  ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
3494 }
3495 
3505 static int ixgbe_write_uc_addr_list(struct net_device *netdev)
3506 {
3507  struct ixgbe_adapter *adapter = netdev_priv(netdev);
3508  struct ixgbe_hw *hw = &adapter->hw;
3509  unsigned int rar_entries = hw->mac.num_rar_entries - 1;
3510  int count = 0;
3511 
3512  /* In SR-IOV mode significantly less RAR entries are available */
3513  if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3514  rar_entries = IXGBE_MAX_PF_MACVLANS - 1;
3515 
3516  /* return ENOMEM indicating insufficient memory for addresses */
3517  if (netdev_uc_count(netdev) > rar_entries)
3518  return -ENOMEM;
3519 
3520  if (!netdev_uc_empty(netdev)) {
3521  struct netdev_hw_addr *ha;
3522  /* return error if we do not support writing to RAR table */
3523  if (!hw->mac.ops.set_rar)
3524  return -ENOMEM;
3525 
3526  netdev_for_each_uc_addr(ha, netdev) {
3527  if (!rar_entries)
3528  break;
3529  hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
3530  VMDQ_P(0), IXGBE_RAH_AV);
3531  count++;
3532  }
3533  }
3534  /* write the addresses in reverse order to avoid write combining */
3535  for (; rar_entries > 0 ; rar_entries--)
3536  hw->mac.ops.clear_rar(hw, rar_entries);
3537 
3538  return count;
3539 }
3540 
3550 void ixgbe_set_rx_mode(struct net_device *netdev)
3551 {
3552  struct ixgbe_adapter *adapter = netdev_priv(netdev);
3553  struct ixgbe_hw *hw = &adapter->hw;
3554  u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
3555  int count;
3556 
3557  /* Check for Promiscuous and All Multicast modes */
3558 
3559  fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3560 
3561  /* set all bits that we expect to always be set */
3562  fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */
3563  fctrl |= IXGBE_FCTRL_BAM;
3564  fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
3565  fctrl |= IXGBE_FCTRL_PMCF;
3566 
3567  /* clear the bits we are changing the status of */
3568  fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3569 
3570  if (netdev->flags & IFF_PROMISC) {
3571  hw->addr_ctrl.user_set_promisc = true;
3572  fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3573  vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
3574  /* don't hardware filter vlans in promisc mode */
3575  ixgbe_vlan_filter_disable(adapter);
3576  } else {
3577  if (netdev->flags & IFF_ALLMULTI) {
3578  fctrl |= IXGBE_FCTRL_MPE;
3579  vmolr |= IXGBE_VMOLR_MPE;
3580  } else {
3581  /*
3582  * Write addresses to the MTA, if the attempt fails
3583  * then we should just turn on promiscuous mode so
3584  * that we can at least receive multicast traffic
3585  */
3586  hw->mac.ops.update_mc_addr_list(hw, netdev);
3587  vmolr |= IXGBE_VMOLR_ROMPE;
3588  }
3589  ixgbe_vlan_filter_enable(adapter);
3590  hw->addr_ctrl.user_set_promisc = false;
3591  }
3592 
3593  /*
3594  * Write addresses to available RAR registers, if there is not
3595  * sufficient space to store all the addresses then enable
3596  * unicast promiscuous mode
3597  */
3598  count = ixgbe_write_uc_addr_list(netdev);
3599  if (count < 0) {
3600  fctrl |= IXGBE_FCTRL_UPE;
3601  vmolr |= IXGBE_VMOLR_ROPE;
3602  }
3603 
3604  if (adapter->num_vfs)
3605  ixgbe_restore_vf_multicasts(adapter);
3606 
3607  if (hw->mac.type != ixgbe_mac_82598EB) {
3608  vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &
3611  IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);
3612  }
3613 
3614  /* This is useful for sniffing bad packets. */
3615  if (adapter->netdev->features & NETIF_F_RXALL) {
3616  /* UPE and MPE will be handled by normal PROMISC logic
3617  * in e1000e_set_rx_mode */
3618  fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */
3619  IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */
3620  IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */
3621 
3622  fctrl &= ~(IXGBE_FCTRL_DPF);
3623  /* NOTE: VLAN filtering is disabled by setting PROMISC */
3624  }
3625 
3626  IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3627 
3628  if (netdev->features & NETIF_F_HW_VLAN_RX)
3629  ixgbe_vlan_strip_enable(adapter);
3630  else
3631  ixgbe_vlan_strip_disable(adapter);
3632 }
3633 
3634 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
3635 {
3636  int q_idx;
3637 
3638  for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
3639  napi_enable(&adapter->q_vector[q_idx]->napi);
3640 }
3641 
3642 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
3643 {
3644  int q_idx;
3645 
3646  for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
3647  napi_disable(&adapter->q_vector[q_idx]->napi);
3648 }
3649 
3650 #ifdef CONFIG_IXGBE_DCB
3651 
3659 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
3660 {
3661  struct ixgbe_hw *hw = &adapter->hw;
3662  int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3663 
3664  if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
3665  if (hw->mac.type == ixgbe_mac_82598EB)
3666  netif_set_gso_max_size(adapter->netdev, 65536);
3667  return;
3668  }
3669 
3670  if (hw->mac.type == ixgbe_mac_82598EB)
3671  netif_set_gso_max_size(adapter->netdev, 32768);
3672 
3673 #ifdef IXGBE_FCOE
3674  if (adapter->netdev->features & NETIF_F_FCOE_MTU)
3675  max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
3676 #endif
3677 
3678  /* reconfigure the hardware */
3679  if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
3680  ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3681  DCB_TX_CONFIG);
3682  ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3683  DCB_RX_CONFIG);
3684  ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
3685  } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
3686  ixgbe_dcb_hw_ets(&adapter->hw,
3687  adapter->ixgbe_ieee_ets,
3688  max_frame);
3689  ixgbe_dcb_hw_pfc_config(&adapter->hw,
3690  adapter->ixgbe_ieee_pfc->pfc_en,
3691  adapter->ixgbe_ieee_ets->prio_tc);
3692  }
3693 
3694  /* Enable RSS Hash per TC */
3695  if (hw->mac.type != ixgbe_mac_82598EB) {
3696  u32 msb = 0;
3697  u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;
3698 
3699  while (rss_i) {
3700  msb++;
3701  rss_i >>= 1;
3702  }
3703 
3704  /* write msb to all 8 TCs in one write */
3705  IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);
3706  }
3707 }
3708 #endif
3709 
3710 /* Additional bittime to account for IXGBE framing */
3711 #define IXGBE_ETH_FRAMING 20
3712 
3719 static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
3720 {
3721  struct ixgbe_hw *hw = &adapter->hw;
3722  struct net_device *dev = adapter->netdev;
3723  int link, tc, kb, marker;
3724  u32 dv_id, rx_pba;
3725 
3726  /* Calculate max LAN frame size */
3727  tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
3728 
3729 #ifdef IXGBE_FCOE
3730  /* FCoE traffic class uses FCOE jumbo frames */
3731  if ((dev->features & NETIF_F_FCOE_MTU) &&
3732  (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) &&
3733  (pb == ixgbe_fcoe_get_tc(adapter)))
3734  tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
3735 
3736 #endif
3737  /* Calculate delay value for device */
3738  switch (hw->mac.type) {
3739  case ixgbe_mac_X540:
3740  dv_id = IXGBE_DV_X540(link, tc);
3741  break;
3742  default:
3743  dv_id = IXGBE_DV(link, tc);
3744  break;
3745  }
3746 
3747  /* Loopback switch introduces additional latency */
3748  if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3749  dv_id += IXGBE_B2BT(tc);
3750 
3751  /* Delay value is calculated in bit times convert to KB */
3752  kb = IXGBE_BT2KB(dv_id);
3753  rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
3754 
3755  marker = rx_pba - kb;
3756 
3757  /* It is possible that the packet buffer is not large enough
3758  * to provide required headroom. In this case throw an error
3759  * to user and a do the best we can.
3760  */
3761  if (marker < 0) {
3762  e_warn(drv, "Packet Buffer(%i) can not provide enough"
3763  "headroom to support flow control."
3764  "Decrease MTU or number of traffic classes\n", pb);
3765  marker = tc + 1;
3766  }
3767 
3768  return marker;
3769 }
3770 
3777 static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter)
3778 {
3779  struct ixgbe_hw *hw = &adapter->hw;
3780  struct net_device *dev = adapter->netdev;
3781  int tc;
3782  u32 dv_id;
3783 
3784  /* Calculate max LAN frame size */
3785  tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
3786 
3787  /* Calculate delay value for device */
3788  switch (hw->mac.type) {
3789  case ixgbe_mac_X540:
3790  dv_id = IXGBE_LOW_DV_X540(tc);
3791  break;
3792  default:
3793  dv_id = IXGBE_LOW_DV(tc);
3794  break;
3795  }
3796 
3797  /* Delay value is calculated in bit times convert to KB */
3798  return IXGBE_BT2KB(dv_id);
3799 }
3800 
3801 /*
3802  * ixgbe_pbthresh_setup - calculate and setup high low water marks
3803  */
3804 static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
3805 {
3806  struct ixgbe_hw *hw = &adapter->hw;
3807  int num_tc = netdev_get_num_tc(adapter->netdev);
3808  int i;
3809 
3810  if (!num_tc)
3811  num_tc = 1;
3812 
3813  hw->fc.low_water = ixgbe_lpbthresh(adapter);
3814 
3815  for (i = 0; i < num_tc; i++) {
3816  hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);
3817 
3818  /* Low water marks must not be larger than high water marks */
3819  if (hw->fc.low_water > hw->fc.high_water[i])
3820  hw->fc.low_water = 0;
3821  }
3822 }
3823 
3824 static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
3825 {
3826  struct ixgbe_hw *hw = &adapter->hw;
3827  int hdrm;
3828  u8 tc = netdev_get_num_tc(adapter->netdev);
3829 
3830  if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
3832  hdrm = 32 << adapter->fdir_pballoc;
3833  else
3834  hdrm = 0;
3835 
3836  hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
3837  ixgbe_pbthresh_setup(adapter);
3838 }
3839 
3840 static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
3841 {
3842  struct ixgbe_hw *hw = &adapter->hw;
3843  struct hlist_node *node, *node2;
3844  struct ixgbe_fdir_filter *filter;
3845 
3846  spin_lock(&adapter->fdir_perfect_lock);
3847 
3848  if (!hlist_empty(&adapter->fdir_filter_list))
3850 
3851  hlist_for_each_entry_safe(filter, node, node2,
3852  &adapter->fdir_filter_list, fdir_node) {
3854  &filter->filter,
3855  filter->sw_idx,
3856  (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
3858  adapter->rx_ring[filter->action]->reg_idx);
3859  }
3860 
3861  spin_unlock(&adapter->fdir_perfect_lock);
3862 }
3863 
3864 static void ixgbe_configure(struct ixgbe_adapter *adapter)
3865 {
3866  struct ixgbe_hw *hw = &adapter->hw;
3867 
3868  ixgbe_configure_pb(adapter);
3869 #ifdef CONFIG_IXGBE_DCB
3870  ixgbe_configure_dcb(adapter);
3871 #endif
3872  /*
3873  * We must restore virtualization before VLANs or else
3874  * the VLVF registers will not be populated
3875  */
3876  ixgbe_configure_virtualization(adapter);
3877 
3878  ixgbe_set_rx_mode(adapter->netdev);
3879  ixgbe_restore_vlan(adapter);
3880 
3881  switch (hw->mac.type) {
3882  case ixgbe_mac_82599EB:
3883  case ixgbe_mac_X540:
3884  hw->mac.ops.disable_rx_buff(hw);
3885  break;
3886  default:
3887  break;
3888  }
3889 
3890  if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3892  adapter->fdir_pballoc);
3893  } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
3895  adapter->fdir_pballoc);
3896  ixgbe_fdir_filter_restore(adapter);
3897  }
3898 
3899  switch (hw->mac.type) {
3900  case ixgbe_mac_82599EB:
3901  case ixgbe_mac_X540:
3902  hw->mac.ops.enable_rx_buff(hw);
3903  break;
3904  default:
3905  break;
3906  }
3907 
3908 #ifdef IXGBE_FCOE
3909  /* configure FCoE L2 filters, redirection table, and Rx control */
3910  ixgbe_configure_fcoe(adapter);
3911 
3912 #endif /* IXGBE_FCOE */
3913  ixgbe_configure_tx(adapter);
3914  ixgbe_configure_rx(adapter);
3915 }
3916 
3917 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
3918 {
3919  switch (hw->phy.type) {
3920  case ixgbe_phy_sfp_avago:
3921  case ixgbe_phy_sfp_ftl:
3922  case ixgbe_phy_sfp_intel:
3923  case ixgbe_phy_sfp_unknown:
3928  return true;
3929  case ixgbe_phy_nl:
3930  if (hw->mac.type == ixgbe_mac_82598EB)
3931  return true;
3932  default:
3933  return false;
3934  }
3935 }
3936 
3941 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
3942 {
3943  /*
3944  * We are assuming the worst case scenario here, and that
3945  * is that an SFP was inserted/removed after the reset
3946  * but before SFP detection was enabled. As such the best
3947  * solution is to just start searching as soon as we start
3948  */
3949  if (adapter->hw.mac.type == ixgbe_mac_82598EB)
3950  adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
3951 
3952  adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
3953 }
3954 
3961 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
3962 {
3963  u32 autoneg;
3964  bool negotiation, link_up = false;
3965  u32 ret = IXGBE_ERR_LINK_SETUP;
3966 
3967  if (hw->mac.ops.check_link)
3968  ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
3969 
3970  if (ret)
3971  goto link_cfg_out;
3972 
3973  autoneg = hw->phy.autoneg_advertised;
3974  if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3975  ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
3976  &negotiation);
3977  if (ret)
3978  goto link_cfg_out;
3979 
3980  if (hw->mac.ops.setup_link)
3981  ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
3982 link_cfg_out:
3983  return ret;
3984 }
3985 
3986 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
3987 {
3988  struct ixgbe_hw *hw = &adapter->hw;
3989  u32 gpie = 0;
3990 
3991  if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3994  gpie |= IXGBE_GPIE_EIAME;
3995  /*
3996  * use EIAM to auto-mask when MSI-X interrupt is asserted
3997  * this saves a register write for every interrupt
3998  */
3999  switch (hw->mac.type) {
4000  case ixgbe_mac_82598EB:
4002  break;
4003  case ixgbe_mac_82599EB:
4004  case ixgbe_mac_X540:
4005  default:
4006  IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
4007  IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
4008  break;
4009  }
4010  } else {
4011  /* legacy interrupts, use EIAM to auto-mask when reading EICR,
4012  * specifically only auto mask tx and rx interrupts */
4014  }
4015 
4016  /* XXX: to interrupt immediately for EICS writes, enable this */
4017  /* gpie |= IXGBE_GPIE_EIMEN; */
4018 
4019  if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
4020  gpie &= ~IXGBE_GPIE_VTMODE_MASK;
4021 
4022  switch (adapter->ring_feature[RING_F_VMDQ].mask) {
4024  gpie |= IXGBE_GPIE_VTMODE_16;
4025  break;
4027  gpie |= IXGBE_GPIE_VTMODE_32;
4028  break;
4029  default:
4030  gpie |= IXGBE_GPIE_VTMODE_64;
4031  break;
4032  }
4033  }
4034 
4035  /* Enable Thermal over heat sensor interrupt */
4036  if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
4037  switch (adapter->hw.mac.type) {
4038  case ixgbe_mac_82599EB:
4039  gpie |= IXGBE_SDP0_GPIEN;
4040  break;
4041  case ixgbe_mac_X540:
4042  gpie |= IXGBE_EIMS_TS;
4043  break;
4044  default:
4045  break;
4046  }
4047  }
4048 
4049  /* Enable fan failure interrupt */
4050  if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
4051  gpie |= IXGBE_SDP1_GPIEN;
4052 
4053  if (hw->mac.type == ixgbe_mac_82599EB) {
4054  gpie |= IXGBE_SDP1_GPIEN;
4055  gpie |= IXGBE_SDP2_GPIEN;
4056  }
4057 
4058  IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4059 }
4060 
4061 static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
4062 {
4063  struct ixgbe_hw *hw = &adapter->hw;
4064  int err;
4065  u32 ctrl_ext;
4066 
4067  ixgbe_get_hw_control(adapter);
4068  ixgbe_setup_gpie(adapter);
4069 
4070  if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
4071  ixgbe_configure_msix(adapter);
4072  else
4073  ixgbe_configure_msi_and_legacy(adapter);
4074 
4075  /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */
4076  if (hw->mac.ops.enable_tx_laser &&
4077  ((hw->phy.multispeed_fiber) ||
4078  ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
4079  (hw->mac.type == ixgbe_mac_82599EB))))
4080  hw->mac.ops.enable_tx_laser(hw);
4081 
4082  clear_bit(__IXGBE_DOWN, &adapter->state);
4083  ixgbe_napi_enable_all(adapter);
4084 
4085  if (ixgbe_is_sfp(hw)) {
4086  ixgbe_sfp_link_config(adapter);
4087  } else {
4088  err = ixgbe_non_sfp_link_config(hw);
4089  if (err)
4090  e_err(probe, "link_config FAILED %d\n", err);
4091  }
4092 
4093  /* clear any pending interrupts, may auto mask */
4095  ixgbe_irq_enable(adapter, true, true);
4096 
4097  /*
4098  * If this adapter has a fan, check to see if we had a failure
4099  * before we enabled the interrupt.
4100  */
4101  if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
4102  u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
4103  if (esdp & IXGBE_ESDP_SDP1)
4104  e_crit(drv, "Fan has stopped, replace the adapter\n");
4105  }
4106 
4107  /* enable transmits */
4108  netif_tx_start_all_queues(adapter->netdev);
4109 
4110  /* bring the link up in the watchdog, this could race with our first
4111  * link up interrupt but shouldn't be a problem */
4112  adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
4113  adapter->link_check_timeout = jiffies;
4114  mod_timer(&adapter->service_timer, jiffies);
4115 
4116  /* Set PF Reset Done bit so PF/VF Mail Ops can work */
4117  ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
4118  ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
4119  IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
4120 }
4121 
4122 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
4123 {
4124  WARN_ON(in_interrupt());
4125  /* put off any impending NetWatchDogTimeout */
4126  adapter->netdev->trans_start = jiffies;
4127 
4128  while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
4129  usleep_range(1000, 2000);
4130  ixgbe_down(adapter);
4131  /*
4132  * If SR-IOV enabled then wait a bit before bringing the adapter
4133  * back up to give the VFs time to respond to the reset. The
4134  * two second wait is based upon the watchdog timer cycle in
4135  * the VF driver.
4136  */
4137  if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4138  msleep(2000);
4139  ixgbe_up(adapter);
4140  clear_bit(__IXGBE_RESETTING, &adapter->state);
4141 }
4142 
4143 void ixgbe_up(struct ixgbe_adapter *adapter)
4144 {
4145  /* hardware has been reset, we need to reload some things */
4146  ixgbe_configure(adapter);
4147 
4148  ixgbe_up_complete(adapter);
4149 }
4150 
4151 void ixgbe_reset(struct ixgbe_adapter *adapter)
4152 {
4153  struct ixgbe_hw *hw = &adapter->hw;
4154  int err;
4155 
4156  /* lock SFP init bit to prevent race conditions with the watchdog */
4157  while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
4158  usleep_range(1000, 2000);
4159 
4160  /* clear all SFP and link config related flags while holding SFP_INIT */
4161  adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
4163  adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
4164 
4165  err = hw->mac.ops.init_hw(hw);
4166  switch (err) {
4167  case 0:
4170  break;
4172  e_dev_err("master disable timed out\n");
4173  break;
4175  /* We are running on a pre-production device, log a warning */
4176  e_dev_warn("This device is a pre-production adapter/LOM. "
4177  "Please be aware there may be issues associated with "
4178  "your hardware. If you are experiencing problems "
4179  "please contact your Intel or hardware "
4180  "representative who provided you with this "
4181  "hardware.\n");
4182  break;
4183  default:
4184  e_dev_err("Hardware Error: %d\n", err);
4185  }
4186 
4187  clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
4188 
4189  /* reprogram the RAR[0] in case user changed it. */
4190  hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV);
4191 
4192  /* update SAN MAC vmdq pool selection */
4193  if (hw->mac.san_mac_rar_index)
4194  hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
4195 }
4196 
4201 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
4202 {
4203  struct device *dev = rx_ring->dev;
4204  unsigned long size;
4205  u16 i;
4206 
4207  /* ring already cleared, nothing to do */
4208  if (!rx_ring->rx_buffer_info)
4209  return;
4210 
4211  /* Free all the Rx ring sk_buffs */
4212  for (i = 0; i < rx_ring->count; i++) {
4213  struct ixgbe_rx_buffer *rx_buffer;
4214 
4215  rx_buffer = &rx_ring->rx_buffer_info[i];
4216  if (rx_buffer->skb) {
4217  struct sk_buff *skb = rx_buffer->skb;
4218  if (IXGBE_CB(skb)->page_released) {
4219  dma_unmap_page(dev,
4220  IXGBE_CB(skb)->dma,
4221  ixgbe_rx_bufsz(rx_ring),
4222  DMA_FROM_DEVICE);
4223  IXGBE_CB(skb)->page_released = false;
4224  }
4225  dev_kfree_skb(skb);
4226  }
4227  rx_buffer->skb = NULL;
4228  if (rx_buffer->dma)
4229  dma_unmap_page(dev, rx_buffer->dma,
4230  ixgbe_rx_pg_size(rx_ring),
4231  DMA_FROM_DEVICE);
4232  rx_buffer->dma = 0;
4233  if (rx_buffer->page)
4234  __free_pages(rx_buffer->page,
4235  ixgbe_rx_pg_order(rx_ring));
4236  rx_buffer->page = NULL;
4237  }
4238 
4239  size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
4240  memset(rx_ring->rx_buffer_info, 0, size);
4241 
4242  /* Zero out the descriptor ring */
4243  memset(rx_ring->desc, 0, rx_ring->size);
4244 
4245  rx_ring->next_to_alloc = 0;
4246  rx_ring->next_to_clean = 0;
4247  rx_ring->next_to_use = 0;
4248 }
4249 
4254 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
4255 {
4256  struct ixgbe_tx_buffer *tx_buffer_info;
4257  unsigned long size;
4258  u16 i;
4259 
4260  /* ring already cleared, nothing to do */
4261  if (!tx_ring->tx_buffer_info)
4262  return;
4263 
4264  /* Free all the Tx ring sk_buffs */
4265  for (i = 0; i < tx_ring->count; i++) {
4266  tx_buffer_info = &tx_ring->tx_buffer_info[i];
4267  ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4268  }
4269 
4270  netdev_tx_reset_queue(txring_txq(tx_ring));
4271 
4272  size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
4273  memset(tx_ring->tx_buffer_info, 0, size);
4274 
4275  /* Zero out the descriptor ring */
4276  memset(tx_ring->desc, 0, tx_ring->size);
4277 
4278  tx_ring->next_to_use = 0;
4279  tx_ring->next_to_clean = 0;
4280 }
4281 
4286 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
4287 {
4288  int i;
4289 
4290  for (i = 0; i < adapter->num_rx_queues; i++)
4291  ixgbe_clean_rx_ring(adapter->rx_ring[i]);
4292 }
4293 
4298 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
4299 {
4300  int i;
4301 
4302  for (i = 0; i < adapter->num_tx_queues; i++)
4303  ixgbe_clean_tx_ring(adapter->tx_ring[i]);
4304 }
4305 
4306 static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
4307 {
4308  struct hlist_node *node, *node2;
4309  struct ixgbe_fdir_filter *filter;
4310 
4311  spin_lock(&adapter->fdir_perfect_lock);
4312 
4313  hlist_for_each_entry_safe(filter, node, node2,
4314  &adapter->fdir_filter_list, fdir_node) {
4315  hlist_del(&filter->fdir_node);
4316  kfree(filter);
4317  }
4318  adapter->fdir_filter_count = 0;
4319 
4320  spin_unlock(&adapter->fdir_perfect_lock);
4321 }
4322 
4323 void ixgbe_down(struct ixgbe_adapter *adapter)
4324 {
4325  struct net_device *netdev = adapter->netdev;
4326  struct ixgbe_hw *hw = &adapter->hw;
4327  u32 rxctrl;
4328  int i;
4329 
4330  /* signal that we are down to the interrupt handler */
4331  set_bit(__IXGBE_DOWN, &adapter->state);
4332 
4333  /* disable receives */
4334  rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4336 
4337  /* disable all enabled rx queues */
4338  for (i = 0; i < adapter->num_rx_queues; i++)
4339  /* this call also flushes the previous write */
4340  ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
4341 
4342  usleep_range(10000, 20000);
4343 
4344  netif_tx_stop_all_queues(netdev);
4345 
4346  /* call carrier off first to avoid false dev_watchdog timeouts */
4347  netif_carrier_off(netdev);
4348  netif_tx_disable(netdev);
4349 
4350  ixgbe_irq_disable(adapter);
4351 
4352  ixgbe_napi_disable_all(adapter);
4353 
4356  adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
4357 
4358  del_timer_sync(&adapter->service_timer);
4359 
4360  if (adapter->num_vfs) {
4361  /* Clear EITR Select mapping */
4362  IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
4363 
4364  /* Mark all the VFs as inactive */
4365  for (i = 0 ; i < adapter->num_vfs; i++)
4366  adapter->vfinfo[i].clear_to_send = false;
4367 
4368  /* ping all the active vfs to let them know we are going down */
4369  ixgbe_ping_all_vfs(adapter);
4370 
4371  /* Disable all VFTE/VFRE TX/RX */
4372  ixgbe_disable_tx_rx(adapter);
4373  }
4374 
4375  /* disable transmits in the hardware now that interrupts are off */
4376  for (i = 0; i < adapter->num_tx_queues; i++) {
4377  u8 reg_idx = adapter->tx_ring[i]->reg_idx;
4379  }
4380 
4381  /* Disable the Tx DMA engine on 82599 and X540 */
4382  switch (hw->mac.type) {
4383  case ixgbe_mac_82599EB:
4384  case ixgbe_mac_X540:
4387  ~IXGBE_DMATXCTL_TE));
4388  break;
4389  default:
4390  break;
4391  }
4392 
4393  if (!pci_channel_offline(adapter->pdev))
4394  ixgbe_reset(adapter);
4395 
4396  /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
4397  if (hw->mac.ops.disable_tx_laser &&
4398  ((hw->phy.multispeed_fiber) ||
4399  ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
4400  (hw->mac.type == ixgbe_mac_82599EB))))
4401  hw->mac.ops.disable_tx_laser(hw);
4402 
4403  ixgbe_clean_all_tx_rings(adapter);
4404  ixgbe_clean_all_rx_rings(adapter);
4405 
4406 #ifdef CONFIG_IXGBE_DCA
4407  /* since we reset the hardware DCA settings were cleared */
4408  ixgbe_setup_dca(adapter);
4409 #endif
4410 }
4411 
4416 static void ixgbe_tx_timeout(struct net_device *netdev)
4417 {
4418  struct ixgbe_adapter *adapter = netdev_priv(netdev);
4419 
4420  /* Do the reset outside of interrupt context */
4421  ixgbe_tx_timeout_reset(adapter);
4422 }
4423 
4432 static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
4433 {
4434  struct ixgbe_hw *hw = &adapter->hw;
4435  struct pci_dev *pdev = adapter->pdev;
4436  unsigned int rss;
4437 #ifdef CONFIG_IXGBE_DCB
4438  int j;
4439  struct tc_configuration *tc;
4440 #endif
4441 
4442  /* PCI config space info */
4443 
4444  hw->vendor_id = pdev->vendor;
4445  hw->device_id = pdev->device;
4446  hw->revision_id = pdev->revision;
4449 
4450  /* Set capability flags */
4452  adapter->ring_feature[RING_F_RSS].limit = rss;
4453  switch (hw->mac.type) {
4454  case ixgbe_mac_82598EB:
4455  if (hw->device_id == IXGBE_DEV_ID_82598AT)
4456  adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
4458  break;
4459  case ixgbe_mac_X540:
4461  case ixgbe_mac_82599EB:
4463  adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
4464  adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
4467  /* Flow Director hash filters enabled */
4468  adapter->atr_sample_rate = 20;
4469  adapter->ring_feature[RING_F_FDIR].limit =
4472 #ifdef IXGBE_FCOE
4473  adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
4474  adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
4475 #ifdef CONFIG_IXGBE_DCB
4476  /* Default traffic class to use for FCoE */
4477  adapter->fcoe.up = IXGBE_FCOE_DEFTC;
4478 #endif
4479 #endif /* IXGBE_FCOE */
4480  break;
4481  default:
4482  break;
4483  }
4484 
4485 #ifdef IXGBE_FCOE
4486  /* FCoE support exists, always init the FCoE lock */
4487  spin_lock_init(&adapter->fcoe.lock);
4488 
4489 #endif
4490  /* n-tuple support exists, always init our spinlock */
4491  spin_lock_init(&adapter->fdir_perfect_lock);
4492 
4493 #ifdef CONFIG_IXGBE_DCB
4494  switch (hw->mac.type) {
4495  case ixgbe_mac_X540:
4496  adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
4497  adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
4498  break;
4499  default:
4500  adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
4501  adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
4502  break;
4503  }
4504 
4505  /* Configure DCB traffic classes */
4506  for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
4507  tc = &adapter->dcb_cfg.tc_config[j];
4508  tc->path[DCB_TX_CONFIG].bwg_id = 0;
4509  tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
4510  tc->path[DCB_RX_CONFIG].bwg_id = 0;
4511  tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
4512  tc->dcb_pfc = pfc_disabled;
4513  }
4514 
4515  /* Initialize default user to priority mapping, UPx->TC0 */
4516  tc = &adapter->dcb_cfg.tc_config[0];
4517  tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
4518  tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
4519 
4520  adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
4521  adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
4522  adapter->dcb_cfg.pfc_mode_enable = false;
4523  adapter->dcb_set_bitmap = 0x00;
4525  memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
4526  sizeof(adapter->temp_dcb_cfg));
4527 
4528 #endif
4529 
4530  /* default flow control settings */
4531  hw->fc.requested_mode = ixgbe_fc_full;
4532  hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
4533  ixgbe_pbthresh_setup(adapter);
4534  hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
4535  hw->fc.send_xon = true;
4536  hw->fc.disable_fc_autoneg = false;
4537 
4538 #ifdef CONFIG_PCI_IOV
4539  /* assign number of SR-IOV VFs */
4540  if (hw->mac.type != ixgbe_mac_82598EB)
4541  adapter->num_vfs = (max_vfs > 63) ? 0 : max_vfs;
4542 
4543 #endif
4544  /* enable itr by default in dynamic mode */
4545  adapter->rx_itr_setting = 1;
4546  adapter->tx_itr_setting = 1;
4547 
4548  /* set default ring sizes */
4549  adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
4550  adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
4551 
4552  /* set default work limits */
4554 
4555  /* initialize eeprom parameters */
4557  e_dev_err("EEPROM initialization failed\n");
4558  return -EIO;
4559  }
4560 
4561  set_bit(__IXGBE_DOWN, &adapter->state);
4562 
4563  return 0;
4564 }
4565 
4573 {
4574  struct device *dev = tx_ring->dev;
4575  int orig_node = dev_to_node(dev);
4576  int numa_node = -1;
4577  int size;
4578 
4579  size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
4580 
4581  if (tx_ring->q_vector)
4582  numa_node = tx_ring->q_vector->numa_node;
4583 
4584  tx_ring->tx_buffer_info = vzalloc_node(size, numa_node);
4585  if (!tx_ring->tx_buffer_info)
4586  tx_ring->tx_buffer_info = vzalloc(size);
4587  if (!tx_ring->tx_buffer_info)
4588  goto err;
4589 
4590  /* round up to nearest 4K */
4591  tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
4592  tx_ring->size = ALIGN(tx_ring->size, 4096);
4593 
4594  set_dev_node(dev, numa_node);
4595  tx_ring->desc = dma_alloc_coherent(dev,
4596  tx_ring->size,
4597  &tx_ring->dma,
4598  GFP_KERNEL);
4599  set_dev_node(dev, orig_node);
4600  if (!tx_ring->desc)
4601  tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
4602  &tx_ring->dma, GFP_KERNEL);
4603  if (!tx_ring->desc)
4604  goto err;
4605 
4606  tx_ring->next_to_use = 0;
4607  tx_ring->next_to_clean = 0;
4608  return 0;
4609 
4610 err:
4611  vfree(tx_ring->tx_buffer_info);
4612  tx_ring->tx_buffer_info = NULL;
4613  dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
4614  return -ENOMEM;
4615 }
4616 
4627 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
4628 {
4629  int i, err = 0;
4630 
4631  for (i = 0; i < adapter->num_tx_queues; i++) {
4632  err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
4633  if (!err)
4634  continue;
4635 
4636  e_err(probe, "Allocation for Tx Queue %u failed\n", i);
4637  goto err_setup_tx;
4638  }
4639 
4640  return 0;
4641 err_setup_tx:
4642  /* rewind the index freeing the rings as we go */
4643  while (i--)
4644  ixgbe_free_tx_resources(adapter->tx_ring[i]);
4645  return err;
4646 }
4647 
4655 {
4656  struct device *dev = rx_ring->dev;
4657  int orig_node = dev_to_node(dev);
4658  int numa_node = -1;
4659  int size;
4660 
4661  size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
4662 
4663  if (rx_ring->q_vector)
4664  numa_node = rx_ring->q_vector->numa_node;
4665 
4666  rx_ring->rx_buffer_info = vzalloc_node(size, numa_node);
4667  if (!rx_ring->rx_buffer_info)
4668  rx_ring->rx_buffer_info = vzalloc(size);
4669  if (!rx_ring->rx_buffer_info)
4670  goto err;
4671 
4672  /* Round up to nearest 4K */
4673  rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
4674  rx_ring->size = ALIGN(rx_ring->size, 4096);
4675 
4676  set_dev_node(dev, numa_node);
4677  rx_ring->desc = dma_alloc_coherent(dev,
4678  rx_ring->size,
4679  &rx_ring->dma,
4680  GFP_KERNEL);
4681  set_dev_node(dev, orig_node);
4682  if (!rx_ring->desc)
4683  rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
4684  &rx_ring->dma, GFP_KERNEL);
4685  if (!rx_ring->desc)
4686  goto err;
4687 
4688  rx_ring->next_to_clean = 0;
4689  rx_ring->next_to_use = 0;
4690 
4691  return 0;
4692 err:
4693  vfree(rx_ring->rx_buffer_info);
4694  rx_ring->rx_buffer_info = NULL;
4695  dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
4696  return -ENOMEM;
4697 }
4698 
4709 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
4710 {
4711  int i, err = 0;
4712 
4713  for (i = 0; i < adapter->num_rx_queues; i++) {
4714  err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
4715  if (!err)
4716  continue;
4717 
4718  e_err(probe, "Allocation for Rx Queue %u failed\n", i);
4719  goto err_setup_rx;
4720  }
4721 
4722 #ifdef IXGBE_FCOE
4723  err = ixgbe_setup_fcoe_ddp_resources(adapter);
4724  if (!err)
4725 #endif
4726  return 0;
4727 err_setup_rx:
4728  /* rewind the index freeing the rings as we go */
4729  while (i--)
4730  ixgbe_free_rx_resources(adapter->rx_ring[i]);
4731  return err;
4732 }
4733 
4741 {
4742  ixgbe_clean_tx_ring(tx_ring);
4743 
4744  vfree(tx_ring->tx_buffer_info);
4745  tx_ring->tx_buffer_info = NULL;
4746 
4747  /* if not set, then don't free */
4748  if (!tx_ring->desc)
4749  return;
4750 
4751  dma_free_coherent(tx_ring->dev, tx_ring->size,
4752  tx_ring->desc, tx_ring->dma);
4753 
4754  tx_ring->desc = NULL;
4755 }
4756 
4763 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
4764 {
4765  int i;
4766 
4767  for (i = 0; i < adapter->num_tx_queues; i++)
4768  if (adapter->tx_ring[i]->desc)
4769  ixgbe_free_tx_resources(adapter->tx_ring[i]);
4770 }
4771 
4779 {
4780  ixgbe_clean_rx_ring(rx_ring);
4781 
4782  vfree(rx_ring->rx_buffer_info);
4783  rx_ring->rx_buffer_info = NULL;
4784 
4785  /* if not set, then don't free */
4786  if (!rx_ring->desc)
4787  return;
4788 
4789  dma_free_coherent(rx_ring->dev, rx_ring->size,
4790  rx_ring->desc, rx_ring->dma);
4791 
4792  rx_ring->desc = NULL;
4793 }
4794 
4801 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
4802 {
4803  int i;
4804 
4805 #ifdef IXGBE_FCOE
4807 
4808 #endif
4809  for (i = 0; i < adapter->num_rx_queues; i++)
4810  if (adapter->rx_ring[i]->desc)
4811  ixgbe_free_rx_resources(adapter->rx_ring[i]);
4812 }
4813 
4821 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
4822 {
4823  struct ixgbe_adapter *adapter = netdev_priv(netdev);
4824  int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4825 
4826  /* MTU < 68 is an error and causes problems on some kernels */
4827  if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
4828  return -EINVAL;
4829 
4830  /*
4831  * For 82599EB we cannot allow PF to change MTU greater than 1500
4832  * in SR-IOV mode as it may cause buffer overruns in guest VFs that
4833  * don't allocate and chain buffers correctly.
4834  */
4835  if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
4836  (adapter->hw.mac.type == ixgbe_mac_82599EB) &&
4837  (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
4838  return -EINVAL;
4839 
4840  e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
4841 
4842  /* must set new MTU before calling down or up */
4843  netdev->mtu = new_mtu;
4844 
4845  if (netif_running(netdev))
4846  ixgbe_reinit_locked(adapter);
4847 
4848  return 0;
4849 }
4850 
4863 static int ixgbe_open(struct net_device *netdev)
4864 {
4865  struct ixgbe_adapter *adapter = netdev_priv(netdev);
4866  int err;
4867 
4868  /* disallow open during test */
4869  if (test_bit(__IXGBE_TESTING, &adapter->state))
4870  return -EBUSY;
4871 
4872  netif_carrier_off(netdev);
4873 
4874  /* allocate transmit descriptors */
4875  err = ixgbe_setup_all_tx_resources(adapter);
4876  if (err)
4877  goto err_setup_tx;
4878 
4879  /* allocate receive descriptors */
4880  err = ixgbe_setup_all_rx_resources(adapter);
4881  if (err)
4882  goto err_setup_rx;
4883 
4884  ixgbe_configure(adapter);
4885 
4886  err = ixgbe_request_irq(adapter);
4887  if (err)
4888  goto err_req_irq;
4889 
4890  /* Notify the stack of the actual queue counts. */
4891  err = netif_set_real_num_tx_queues(netdev,
4892  adapter->num_rx_pools > 1 ? 1 :
4893  adapter->num_tx_queues);
4894  if (err)
4895  goto err_set_queues;
4896 
4897 
4898  err = netif_set_real_num_rx_queues(netdev,
4899  adapter->num_rx_pools > 1 ? 1 :
4900  adapter->num_rx_queues);
4901  if (err)
4902  goto err_set_queues;
4903 
4904  ixgbe_up_complete(adapter);
4905 
4906  return 0;
4907 
4908 err_set_queues:
4909  ixgbe_free_irq(adapter);
4910 err_req_irq:
4911  ixgbe_free_all_rx_resources(adapter);
4912 err_setup_rx:
4913  ixgbe_free_all_tx_resources(adapter);
4914 err_setup_tx:
4915  ixgbe_reset(adapter);
4916 
4917  return err;
4918 }
4919 
4931 static int ixgbe_close(struct net_device *netdev)
4932 {
4933  struct ixgbe_adapter *adapter = netdev_priv(netdev);
4934 
4935  ixgbe_down(adapter);
4936  ixgbe_free_irq(adapter);
4937 
4938  ixgbe_fdir_filter_exit(adapter);
4939 
4940  ixgbe_free_all_tx_resources(adapter);
4941  ixgbe_free_all_rx_resources(adapter);
4942 
4943  ixgbe_release_hw_control(adapter);
4944 
4945  return 0;
4946 }
4947 
4948 #ifdef CONFIG_PM
4949 static int ixgbe_resume(struct pci_dev *pdev)
4950 {
4951  struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
4952  struct net_device *netdev = adapter->netdev;
4953  u32 err;
4954 
4955  pci_set_power_state(pdev, PCI_D0);
4956  pci_restore_state(pdev);
4957  /*
4958  * pci_restore_state clears dev->state_saved so call
4959  * pci_save_state to restore it.
4960  */
4961  pci_save_state(pdev);
4962 
4963  err = pci_enable_device_mem(pdev);
4964  if (err) {
4965  e_dev_err("Cannot enable PCI device from suspend\n");
4966  return err;
4967  }
4968  pci_set_master(pdev);
4969 
4970  pci_wake_from_d3(pdev, false);
4971 
4972  ixgbe_reset(adapter);
4973 
4974  IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
4975 
4976  rtnl_lock();
4977  err = ixgbe_init_interrupt_scheme(adapter);
4978  if (!err && netif_running(netdev))
4979  err = ixgbe_open(netdev);
4980 
4981  rtnl_unlock();
4982 
4983  if (err)
4984  return err;
4985 
4986  netif_device_attach(netdev);
4987 
4988  return 0;
4989 }
4990 #endif /* CONFIG_PM */
4991 
4992 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
4993 {
4994  struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
4995  struct net_device *netdev = adapter->netdev;
4996  struct ixgbe_hw *hw = &adapter->hw;
4997  u32 ctrl, fctrl;
4998  u32 wufc = adapter->wol;
4999 #ifdef CONFIG_PM
5000  int retval = 0;
5001 #endif
5002 
5003  netif_device_detach(netdev);
5004 
5005  if (netif_running(netdev)) {
5006  rtnl_lock();
5007  ixgbe_down(adapter);
5008  ixgbe_free_irq(adapter);
5009  ixgbe_free_all_tx_resources(adapter);
5010  ixgbe_free_all_rx_resources(adapter);
5011  rtnl_unlock();
5012  }
5013 
5015 
5016 #ifdef CONFIG_PM
5017  retval = pci_save_state(pdev);
5018  if (retval)
5019  return retval;
5020 
5021 #endif
5022  if (wufc) {
5023  ixgbe_set_rx_mode(netdev);
5024 
5025  /*
5026  * enable the optics for both mult-speed fiber and
5027  * 82599 SFP+ fiber as we can WoL.
5028  */
5029  if (hw->mac.ops.enable_tx_laser &&
5030  (hw->phy.multispeed_fiber ||
5031  (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber &&
5032  hw->mac.type == ixgbe_mac_82599EB)))
5033  hw->mac.ops.enable_tx_laser(hw);
5034 
5035  /* turn on all-multi mode if wake on multicast is enabled */
5036  if (wufc & IXGBE_WUFC_MC) {
5037  fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5038  fctrl |= IXGBE_FCTRL_MPE;
5039  IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5040  }
5041 
5042  ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
5043  ctrl |= IXGBE_CTRL_GIO_DIS;
5044  IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
5045 
5046  IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
5047  } else {
5048  IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
5049  IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
5050  }
5051 
5052  switch (hw->mac.type) {
5053  case ixgbe_mac_82598EB:
5054  pci_wake_from_d3(pdev, false);
5055  break;
5056  case ixgbe_mac_82599EB:
5057  case ixgbe_mac_X540:
5058  pci_wake_from_d3(pdev, !!wufc);
5059  break;
5060  default:
5061  break;
5062  }
5063 
5064  *enable_wake = !!wufc;
5065 
5066  ixgbe_release_hw_control(adapter);
5067 
5068  pci_disable_device(pdev);
5069 
5070  return 0;
5071 }
5072 
5073 #ifdef CONFIG_PM
5074 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
5075 {
5076  int retval;
5077  bool wake;
5078 
5079  retval = __ixgbe_shutdown(pdev, &wake);
5080  if (retval)
5081  return retval;
5082 
5083  if (wake) {
5084  pci_prepare_to_sleep(pdev);
5085  } else {
5086  pci_wake_from_d3(pdev, false);
5088  }
5089 
5090  return 0;
5091 }
5092 #endif /* CONFIG_PM */
5093 
5094 static void ixgbe_shutdown(struct pci_dev *pdev)
5095 {
5096  bool wake;
5097 
5098  __ixgbe_shutdown(pdev, &wake);
5099 
5100  if (system_state == SYSTEM_POWER_OFF) {
5101  pci_wake_from_d3(pdev, wake);
5103  }
5104 }
5105 
5110 void ixgbe_update_stats(struct ixgbe_adapter *adapter)
5111 {
5112  struct net_device *netdev = adapter->netdev;
5113  struct ixgbe_hw *hw = &adapter->hw;
5114  struct ixgbe_hw_stats *hwstats = &adapter->stats;
5115  u64 total_mpc = 0;
5116  u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
5117  u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
5118  u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
5119  u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;
5120 
5121  if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5122  test_bit(__IXGBE_RESETTING, &adapter->state))
5123  return;
5124 
5125  if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
5126  u64 rsc_count = 0;
5127  u64 rsc_flush = 0;
5128  for (i = 0; i < adapter->num_rx_queues; i++) {
5129  rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
5130  rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
5131  }
5132  adapter->rsc_total_count = rsc_count;
5133  adapter->rsc_total_flush = rsc_flush;
5134  }
5135 
5136  for (i = 0; i < adapter->num_rx_queues; i++) {
5137  struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
5138  non_eop_descs += rx_ring->rx_stats.non_eop_descs;
5139  alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
5140  alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
5141  hw_csum_rx_error += rx_ring->rx_stats.csum_err;
5142  bytes += rx_ring->stats.bytes;
5143  packets += rx_ring->stats.packets;
5144  }
5145  adapter->non_eop_descs = non_eop_descs;
5146  adapter->alloc_rx_page_failed = alloc_rx_page_failed;
5147  adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
5148  adapter->hw_csum_rx_error = hw_csum_rx_error;
5149  netdev->stats.rx_bytes = bytes;
5150  netdev->stats.rx_packets = packets;
5151 
5152  bytes = 0;
5153  packets = 0;
5154  /* gather some stats to the adapter struct that are per queue */
5155  for (i = 0; i < adapter->num_tx_queues; i++) {
5156  struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
5157  restart_queue += tx_ring->tx_stats.restart_queue;
5158  tx_busy += tx_ring->tx_stats.tx_busy;
5159  bytes += tx_ring->stats.bytes;
5160  packets += tx_ring->stats.packets;
5161  }
5162  adapter->restart_queue = restart_queue;
5163  adapter->tx_busy = tx_busy;
5164  netdev->stats.tx_bytes = bytes;
5165  netdev->stats.tx_packets = packets;
5166 
5167  hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5168 
5169  /* 8 register reads */
5170  for (i = 0; i < 8; i++) {
5171  /* for packet buffers not used, the register should read 0 */
5172  mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5173  missed_rx += mpc;
5174  hwstats->mpc[i] += mpc;
5175  total_mpc += hwstats->mpc[i];
5176  hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5177  hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5178  switch (hw->mac.type) {
5179  case ixgbe_mac_82598EB:
5180  hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5181  hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5182  hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5183  hwstats->pxonrxc[i] +=
5184  IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5185  break;
5186  case ixgbe_mac_82599EB:
5187  case ixgbe_mac_X540:
5188  hwstats->pxonrxc[i] +=
5190  break;
5191  default:
5192  break;
5193  }
5194  }
5195 
5196  /*16 register reads */
5197  for (i = 0; i < 16; i++) {
5198  hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5199  hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5200  if ((hw->mac.type == ixgbe_mac_82599EB) ||
5201  (hw->mac.type == ixgbe_mac_X540)) {
5202  hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
5203  IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
5204  hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
5205  IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
5206  }
5207  }
5208 
5209  hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5210  /* work around hardware counting issue */
5211  hwstats->gprc -= missed_rx;
5212 
5213  ixgbe_update_xoff_received(adapter);
5214 
5215  /* 82598 hardware only has a 32 bit counter in the high register */
5216  switch (hw->mac.type) {
5217  case ixgbe_mac_82598EB:
5218  hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5219  hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5220  hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5221  hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5222  break;
5223  case ixgbe_mac_X540:
5224  /* OS2BMC stats are X540 only*/
5225  hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
5226  hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
5227  hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
5228  hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
5229  case ixgbe_mac_82599EB:
5230  for (i = 0; i < 16; i++)
5231  adapter->hw_rx_no_dma_resources +=
5232  IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5233  hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
5234  IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
5235  hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
5236  IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
5237  hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
5238  IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
5239  hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5240  hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
5241  hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
5242 #ifdef IXGBE_FCOE
5243  hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5244  hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5245  hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5246  hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5247  hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5248  hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5249  /* Add up per cpu counters for total ddp aloc fail */
5250  if (adapter->fcoe.ddp_pool) {
5251  struct ixgbe_fcoe *fcoe = &adapter->fcoe;
5252  struct ixgbe_fcoe_ddp_pool *ddp_pool;
5253  unsigned int cpu;
5254  u64 noddp = 0, noddp_ext_buff = 0;
5255  for_each_possible_cpu(cpu) {
5256  ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);
5257  noddp += ddp_pool->noddp;
5258  noddp_ext_buff += ddp_pool->noddp_ext_buff;
5259  }
5260  hwstats->fcoe_noddp = noddp;
5262  }
5263 #endif /* IXGBE_FCOE */
5264  break;
5265  default:
5266  break;
5267  }
5268  bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5269  hwstats->bprc += bprc;
5270  hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5271  if (hw->mac.type == ixgbe_mac_82598EB)
5272  hwstats->mprc -= bprc;
5273  hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5274  hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5275  hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5276  hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5277  hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5278  hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5279  hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5280  hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5281  lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5282  hwstats->lxontxc += lxon;
5283  lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5284  hwstats->lxofftxc += lxoff;
5285  hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5286  hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5287  /*
5288  * 82598 errata - tx of flow control packets is included in tx counters
5289  */
5290  xon_off_tot = lxon + lxoff;
5291  hwstats->gptc -= xon_off_tot;
5292  hwstats->mptc -= xon_off_tot;
5293  hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
5294  hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5295  hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5296  hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5297  hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5298  hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5299  hwstats->ptc64 -= xon_off_tot;
5300  hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5301  hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5302  hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5303  hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5304  hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5305  hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5306 
5307  /* Fill out the OS statistics structure */
5308  netdev->stats.multicast = hwstats->mprc;
5309 
5310  /* Rx Errors */
5311  netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
5312  netdev->stats.rx_dropped = 0;
5313  netdev->stats.rx_length_errors = hwstats->rlec;
5314  netdev->stats.rx_crc_errors = hwstats->crcerrs;
5315  netdev->stats.rx_missed_errors = total_mpc;
5316 }
5317 
5322 static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
5323 {
5324  struct ixgbe_hw *hw = &adapter->hw;
5325  int i;
5326 
5327  if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
5328  return;
5329 
5331 
5332  /* if interface is down do nothing */
5333  if (test_bit(__IXGBE_DOWN, &adapter->state))
5334  return;
5335 
5336  /* do nothing if we are not using signature filters */
5337  if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
5338  return;
5339 
5340  adapter->fdir_overflow++;
5341 
5342  if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
5343  for (i = 0; i < adapter->num_tx_queues; i++)
5345  &(adapter->tx_ring[i]->state));
5346  /* re-enable flow director interrupts */
5348  } else {
5349  e_err(probe, "failed to finish FDIR re-initialization, "
5350  "ignored adding FDIR ATR filters\n");
5351  }
5352 }
5353 
5363 static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
5364 {
5365  struct ixgbe_hw *hw = &adapter->hw;
5366  u64 eics = 0;
5367  int i;
5368 
5369  /* If we're down or resetting, just bail */
5370  if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5371  test_bit(__IXGBE_RESETTING, &adapter->state))
5372  return;
5373 
5374  /* Force detection of hung controller */
5375  if (netif_carrier_ok(adapter->netdev)) {
5376  for (i = 0; i < adapter->num_tx_queues; i++)
5377  set_check_for_tx_hang(adapter->tx_ring[i]);
5378  }
5379 
5380  if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
5381  /*
5382  * for legacy and MSI interrupts don't set any bits
5383  * that are enabled for EIAM, because this operation
5384  * would set *both* EIMS and EICS for any bit in EIAM
5385  */
5388  } else {
5389  /* get one bit for every active tx/rx interrupt vector */
5390  for (i = 0; i < adapter->num_q_vectors; i++) {
5391  struct ixgbe_q_vector *qv = adapter->q_vector[i];
5392  if (qv->rx.ring || qv->tx.ring)
5393  eics |= ((u64)1 << i);
5394  }
5395  }
5396 
5397  /* Cause software interrupt to ensure rings are cleaned */
5398  ixgbe_irq_rearm_queues(adapter, eics);
5399 
5400 }
5401 
5407 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
5408 {
5409  struct ixgbe_hw *hw = &adapter->hw;
5410  u32 link_speed = adapter->link_speed;
5411  bool link_up = adapter->link_up;
5412  bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
5413 
5414  if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
5415  return;
5416 
5417  if (hw->mac.ops.check_link) {
5418  hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
5419  } else {
5420  /* always assume link is up, if no check link function */
5421  link_speed = IXGBE_LINK_SPEED_10GB_FULL;
5422  link_up = true;
5423  }
5424 
5425  if (adapter->ixgbe_ieee_pfc)
5426  pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
5427 
5428  if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {
5429  hw->mac.ops.fc_enable(hw);
5430  ixgbe_set_rx_drop_en(adapter);
5431  }
5432 
5433  if (link_up ||
5434  time_after(jiffies, (adapter->link_check_timeout +
5436  adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
5438  IXGBE_WRITE_FLUSH(hw);
5439  }
5440 
5441  adapter->link_up = link_up;
5442  adapter->link_speed = link_speed;
5443 }
5444 
5450 static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
5451 {
5452  struct net_device *netdev = adapter->netdev;
5453  struct ixgbe_hw *hw = &adapter->hw;
5454  u32 link_speed = adapter->link_speed;
5455  bool flow_rx, flow_tx;
5456 
5457  /* only continue if link was previously down */
5458  if (netif_carrier_ok(netdev))
5459  return;
5460 
5461  adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
5462 
5463  switch (hw->mac.type) {
5464  case ixgbe_mac_82598EB: {
5465  u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5466  u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
5467  flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
5468  flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
5469  }
5470  break;
5471  case ixgbe_mac_X540:
5472  case ixgbe_mac_82599EB: {
5473  u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
5474  u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
5475  flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
5476  flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
5477  }
5478  break;
5479  default:
5480  flow_tx = false;
5481  flow_rx = false;
5482  break;
5483  }
5484 
5485 #ifdef CONFIG_IXGBE_PTP
5487 #endif
5488 
5489  e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
5490  (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
5491  "10 Gbps" :
5492  (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
5493  "1 Gbps" :
5494  (link_speed == IXGBE_LINK_SPEED_100_FULL ?
5495  "100 Mbps" :
5496  "unknown speed"))),
5497  ((flow_rx && flow_tx) ? "RX/TX" :
5498  (flow_rx ? "RX" :
5499  (flow_tx ? "TX" : "None"))));
5500 
5501  netif_carrier_on(netdev);
5502  ixgbe_check_vf_rate_limit(adapter);
5503 
5504  /* ping all the active vfs to let them know link has changed */
5505  ixgbe_ping_all_vfs(adapter);
5506 }
5507 
5513 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)
5514 {
5515  struct net_device *netdev = adapter->netdev;
5516  struct ixgbe_hw *hw = &adapter->hw;
5517 
5518  adapter->link_up = false;
5519  adapter->link_speed = 0;
5520 
5521  /* only continue if link was up previously */
5522  if (!netif_carrier_ok(netdev))
5523  return;
5524 
5525  /* poll for SFP+ cable when link is down */
5526  if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
5527  adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
5528 
5529 #ifdef CONFIG_IXGBE_PTP
5531 #endif
5532 
5533  e_info(drv, "NIC Link is Down\n");
5534  netif_carrier_off(netdev);
5535 
5536  /* ping all the active vfs to let them know link has changed */
5537  ixgbe_ping_all_vfs(adapter);
5538 }
5539 
5544 static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
5545 {
5546  int i;
5547  int some_tx_pending = 0;
5548 
5549  if (!netif_carrier_ok(adapter->netdev)) {
5550  for (i = 0; i < adapter->num_tx_queues; i++) {
5551  struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
5552  if (tx_ring->next_to_use != tx_ring->next_to_clean) {
5553  some_tx_pending = 1;
5554  break;
5555  }
5556  }
5557 
5558  if (some_tx_pending) {
5559  /* We've lost link, so the controller stops DMA,
5560  * but we've got queued Tx work that's never going
5561  * to get done, so reset controller to flush Tx.
5562  * (Do the reset outside of interrupt context).
5563  */
5564  adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
5565  }
5566  }
5567 }
5568 
5569 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
5570 {
5571  u32 ssvpc;
5572 
5573  /* Do not perform spoof check for 82598 or if not in IOV mode */
5574  if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
5575  adapter->num_vfs == 0)
5576  return;
5577 
5578  ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
5579 
5580  /*
5581  * ssvpc register is cleared on read, if zero then no
5582  * spoofed packets in the last interval.
5583  */
5584  if (!ssvpc)
5585  return;
5586 
5587  e_warn(drv, "%u Spoofed packets detected\n", ssvpc);
5588 }
5589 
5594 static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
5595 {
5596  /* if interface is down do nothing */
5597  if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5598  test_bit(__IXGBE_RESETTING, &adapter->state))
5599  return;
5600 
5601  ixgbe_watchdog_update_link(adapter);
5602 
5603  if (adapter->link_up)
5604  ixgbe_watchdog_link_is_up(adapter);
5605  else
5606  ixgbe_watchdog_link_is_down(adapter);
5607 
5608  ixgbe_spoof_check(adapter);
5609  ixgbe_update_stats(adapter);
5610 
5611  ixgbe_watchdog_flush_tx(adapter);
5612 }
5613 
5618 static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
5619 {
5620  struct ixgbe_hw *hw = &adapter->hw;
5621  s32 err;
5622 
5623  /* not searching for SFP so there is nothing to do here */
5624  if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
5625  !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
5626  return;
5627 
5628  /* someone else is in init, wait until next service event */
5629  if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
5630  return;
5631 
5632  err = hw->phy.ops.identify_sfp(hw);
5633  if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
5634  goto sfp_out;
5635 
5636  if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
5637  /* If no cable is present, then we need to reset
5638  * the next time we find a good cable. */
5639  adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
5640  }
5641 
5642  /* exit on error */
5643  if (err)
5644  goto sfp_out;
5645 
5646  /* exit if reset not needed */
5647  if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
5648  goto sfp_out;
5649 
5650  adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
5651 
5652  /*
5653  * A module may be identified correctly, but the EEPROM may not have
5654  * support for that module. setup_sfp() will fail in that case, so
5655  * we should not allow that module to load.
5656  */
5657  if (hw->mac.type == ixgbe_mac_82598EB)
5658  err = hw->phy.ops.reset(hw);
5659  else
5660  err = hw->mac.ops.setup_sfp(hw);
5661 
5662  if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
5663  goto sfp_out;
5664 
5665  adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
5666  e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
5667 
5668 sfp_out:
5669  clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
5670 
5671  if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
5672  (adapter->netdev->reg_state == NETREG_REGISTERED)) {
5673  e_dev_err("failed to initialize because an unsupported "
5674  "SFP+ module type was detected.\n");
5675  e_dev_err("Reload the driver after installing a "
5676  "supported module.\n");
5677  unregister_netdev(adapter->netdev);
5678  }
5679 }
5680 
5685 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
5686 {
5687  struct ixgbe_hw *hw = &adapter->hw;
5688  u32 autoneg;
5689  bool negotiation;
5690 
5691  if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
5692  return;
5693 
5694  /* someone else is in init, wait until next service event */
5695  if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
5696  return;
5697 
5698  adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
5699 
5700  autoneg = hw->phy.autoneg_advertised;
5701  if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5702  hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
5703  if (hw->mac.ops.setup_link)
5704  hw->mac.ops.setup_link(hw, autoneg, negotiation, true);
5705 
5706  adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
5707  adapter->link_check_timeout = jiffies;
5708  clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
5709 }
5710 
5711 #ifdef CONFIG_PCI_IOV
5712 static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
5713 {
5714  int vf;
5715  struct ixgbe_hw *hw = &adapter->hw;
5716  struct net_device *netdev = adapter->netdev;
5717  u32 gpc;
5718  u32 ciaa, ciad;
5719 
5720  gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
5721  if (gpc) /* If incrementing then no need for the check below */
5722  return;
5723  /*
5724  * Check to see if a bad DMA write target from an errant or
5725  * malicious VF has caused a PCIe error. If so then we can
5726  * issue a VFLR to the offending VF(s) and then resume without
5727  * requesting a full slot reset.
5728  */
5729 
5730  for (vf = 0; vf < adapter->num_vfs; vf++) {
5731  ciaa = (vf << 16) | 0x80000000;
5732  /* 32 bit read so align, we really want status at offset 6 */
5733  ciaa |= PCI_COMMAND;
5734  IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
5735  ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599);
5736  ciaa &= 0x7FFFFFFF;
5737  /* disable debug mode asap after reading data */
5738  IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
5739  /* Get the upper 16 bits which will be the PCI status reg */
5740  ciad >>= 16;
5741  if (ciad & PCI_STATUS_REC_MASTER_ABORT) {
5742  netdev_err(netdev, "VF %d Hung DMA\n", vf);
5743  /* Issue VFLR */
5744  ciaa = (vf << 16) | 0x80000000;
5745  ciaa |= 0xA8;
5746  IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
5747  ciad = 0x00008000; /* VFLR */
5748  IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad);
5749  ciaa &= 0x7FFFFFFF;
5750  IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
5751  }
5752  }
5753 }
5754 
5755 #endif
5756 
5760 static void ixgbe_service_timer(unsigned long data)
5761 {
5762  struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
5763  unsigned long next_event_offset;
5764  bool ready = true;
5765 
5766  /* poll faster when waiting for link */
5767  if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
5768  next_event_offset = HZ / 10;
5769  else
5770  next_event_offset = HZ * 2;
5771 
5772 #ifdef CONFIG_PCI_IOV
5773  /*
5774  * don't bother with SR-IOV VF DMA hang check if there are
5775  * no VFs or the link is down
5776  */
5777  if (!adapter->num_vfs ||
5778  (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
5779  goto normal_timer_service;
5780 
5781  /* If we have VFs allocated then we must check for DMA hangs */
5782  ixgbe_check_for_bad_vf(adapter);
5783  next_event_offset = HZ / 50;
5784  adapter->timer_event_accumulator++;
5785 
5786  if (adapter->timer_event_accumulator >= 100)
5787  adapter->timer_event_accumulator = 0;
5788  else
5789  ready = false;
5790 
5791 normal_timer_service:
5792 #endif
5793  /* Reset the timer */
5794  mod_timer(&adapter->service_timer, next_event_offset + jiffies);
5795 
5796  if (ready)
5797  ixgbe_service_event_schedule(adapter);
5798 }
5799 
5800 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
5801 {
5802  if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
5803  return;
5804 
5805  adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;
5806 
5807  /* If we're already down or resetting, just bail */
5808  if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5809  test_bit(__IXGBE_RESETTING, &adapter->state))
5810  return;
5811 
5812  ixgbe_dump(adapter);
5813  netdev_err(adapter->netdev, "Reset adapter\n");
5814  adapter->tx_timeout_count++;
5815 
5816  ixgbe_reinit_locked(adapter);
5817 }
5818 
5823 static void ixgbe_service_task(struct work_struct *work)
5824 {
5825  struct ixgbe_adapter *adapter = container_of(work,
5826  struct ixgbe_adapter,
5827  service_task);
5828 
5829  ixgbe_reset_subtask(adapter);
5830  ixgbe_sfp_detection_subtask(adapter);
5831  ixgbe_sfp_link_config_subtask(adapter);
5832  ixgbe_check_overtemp_subtask(adapter);
5833  ixgbe_watchdog_subtask(adapter);
5834  ixgbe_fdir_reinit_subtask(adapter);
5835  ixgbe_check_hang_subtask(adapter);
5836 #ifdef CONFIG_IXGBE_PTP
5837  ixgbe_ptp_overflow_check(adapter);
5838 #endif
5839 
5840  ixgbe_service_event_complete(adapter);
5841 }
5842 
5843 static int ixgbe_tso(struct ixgbe_ring *tx_ring,
5844  struct ixgbe_tx_buffer *first,
5845  u8 *hdr_len)
5846 {
5847  struct sk_buff *skb = first->skb;
5848  u32 vlan_macip_lens, type_tucmd;
5849  u32 mss_l4len_idx, l4len;
5850 
5851  if (!skb_is_gso(skb))
5852  return 0;
5853 
5854  if (skb_header_cloned(skb)) {
5855  int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5856  if (err)
5857  return err;
5858  }
5859 
5860  /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
5861  type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
5862 
5863  if (first->