ixgbe_main.c

Go to the documentation of this file.

/*******************************************************************************

  Intel 10 Gigabit PCI Express Linux driver
  Copyright(c) 1999 - 2012 Intel Corporation.

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information:
  e1000-devel Mailing List <[email protected]>
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

#include <linux/types.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/in.h>
#include <linux/interrupt.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/sctp.h>
#include <linux/pkt_sched.h>
#include <linux/ipv6.h>
#include <linux/slab.h>
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/ethtool.h>
#include <linux/if.h>
#include <linux/if_vlan.h>
#include <linux/prefetch.h>
#include <scsi/fc/fc_fcoe.h>

#include "ixgbe.h"
#include "ixgbe_common.h"
#include "ixgbe_dcb_82599.h"
#include "ixgbe_sriov.h"

char ixgbe_driver_name[] = "ixgbe";
static const char ixgbe_driver_string[] =
                  "Intel(R) 10 Gigabit PCI Express Network Driver";
#ifdef IXGBE_FCOE
char ixgbe_default_device_descr[] =
                  "Intel(R) 10 Gigabit Network Connection";
#else
static char ixgbe_default_device_descr[] =
                  "Intel(R) 10 Gigabit Network Connection";
#endif
#define MAJ 3
#define MIN 9
#define BUILD 15
#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
    __stringify(BUILD) "-k"
const char ixgbe_driver_version[] = DRV_VERSION;
static const char ixgbe_copyright[] =
                "Copyright (c) 1999-2012 Intel Corporation.";

static const struct ixgbe_info *ixgbe_info_tbl[] = {
    [board_82598] = &ixgbe_82598_info,
    [board_82599] = &ixgbe_82599_info,
    [board_X540] = &ixgbe_X540_info,
};

/* ixgbe_pci_tbl - PCI Device ID Table
 *
 * Wildcard entries (PCI_ANY_ID) should come last
 * Last entry must be all 0s
 *
 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
 *   Class, Class Mask, private data (not used) }
 */
static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
    {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
    /* required last entry */
    {0, }
};
MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);

#ifdef CONFIG_IXGBE_DCA
static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
                void *p);
static struct notifier_block dca_notifier = {
    .notifier_call = ixgbe_notify_dca,
    .next          = NULL,
    .priority      = 0
};
#endif

#ifdef CONFIG_PCI_IOV
static unsigned int max_vfs;
module_param(max_vfs, uint, 0);
MODULE_PARM_DESC(max_vfs,
         "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63");
#endif /* CONFIG_PCI_IOV */

static unsigned int allow_unsupported_sfp;
module_param(allow_unsupported_sfp, uint, 0);
MODULE_PARM_DESC(allow_unsupported_sfp,
         "Allow unsupported and untested SFP+ modules on 82599-based adapters");

#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

MODULE_AUTHOR("Intel Corporation, <[email protected]>");
MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
{
    if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
        !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
        schedule_work(&adapter->service_task);
}

static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
{
    BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));

    /* flush memory to make sure state is correct before next watchdog */
    smp_mb__before_clear_bit();
    clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
}

struct ixgbe_reg_info {
    u32 ofs;
    char *name;
};

static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {

    /* General Registers */
    {IXGBE_CTRL, "CTRL"},
    {IXGBE_STATUS, "STATUS"},
    {IXGBE_CTRL_EXT, "CTRL_EXT"},

    /* Interrupt Registers */
    {IXGBE_EICR, "EICR"},

    /* RX Registers */
    {IXGBE_SRRCTL(0), "SRRCTL"},
    {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
    {IXGBE_RDLEN(0), "RDLEN"},
    {IXGBE_RDH(0), "RDH"},
    {IXGBE_RDT(0), "RDT"},
    {IXGBE_RXDCTL(0), "RXDCTL"},
    {IXGBE_RDBAL(0), "RDBAL"},
    {IXGBE_RDBAH(0), "RDBAH"},

    /* TX Registers */
    {IXGBE_TDBAL(0), "TDBAL"},
    {IXGBE_TDBAH(0), "TDBAH"},
    {IXGBE_TDLEN(0), "TDLEN"},
    {IXGBE_TDH(0), "TDH"},
    {IXGBE_TDT(0), "TDT"},
    {IXGBE_TXDCTL(0), "TXDCTL"},

    /* List Terminator */
    {}
};


/*
 * ixgbe_regdump - register printout routine
 */
static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
{
    int i = 0, j = 0;
    char rname[16];
    u32 regs[64];

    switch (reginfo->ofs) {
    case IXGBE_SRRCTL(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
        break;
    case IXGBE_DCA_RXCTRL(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
        break;
    case IXGBE_RDLEN(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
        break;
    case IXGBE_RDH(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
        break;
    case IXGBE_RDT(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
        break;
    case IXGBE_RXDCTL(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
        break;
    case IXGBE_RDBAL(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
        break;
    case IXGBE_RDBAH(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
        break;
    case IXGBE_TDBAL(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
        break;
    case IXGBE_TDBAH(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
        break;
    case IXGBE_TDLEN(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
        break;
    case IXGBE_TDH(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
        break;
    case IXGBE_TDT(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
        break;
    case IXGBE_TXDCTL(0):
        for (i = 0; i < 64; i++)
            regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
        break;
    default:
        pr_info("%-15s %08x\n", reginfo->name,
            IXGBE_READ_REG(hw, reginfo->ofs));
        return;
    }

    for (i = 0; i < 8; i++) {
        snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
        pr_err("%-15s", rname);
        for (j = 0; j < 8; j++)
            pr_cont(" %08x", regs[i*8+j]);
        pr_cont("\n");
    }

}

/*
 * ixgbe_dump - Print registers, tx-rings and rx-rings
 */
static void ixgbe_dump(struct ixgbe_adapter *adapter)
{
    struct net_device *netdev = adapter->netdev;
    struct ixgbe_hw *hw = &adapter->hw;
    struct ixgbe_reg_info *reginfo;
    int n = 0;
    struct ixgbe_ring *tx_ring;
    struct ixgbe_tx_buffer *tx_buffer;
    union ixgbe_adv_tx_desc *tx_desc;
    struct my_u0 { u64 a; u64 b; } *u0;
    struct ixgbe_ring *rx_ring;
    union ixgbe_adv_rx_desc *rx_desc;
    struct ixgbe_rx_buffer *rx_buffer_info;
    u32 staterr;
    int i = 0;

    if (!netif_msg_hw(adapter))
        return;

    /* Print netdevice Info */
    if (netdev) {
        dev_info(&adapter->pdev->dev, "Net device Info\n");
        pr_info("Device Name     state            "
            "trans_start      last_rx\n");
        pr_info("%-15s %016lX %016lX %016lX\n",
            netdev->name,
            netdev->state,
            netdev->trans_start,
            netdev->last_rx);
    }

    /* Print Registers */
    dev_info(&adapter->pdev->dev, "Register Dump\n");
    pr_info(" Register Name   Value\n");
    for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
         reginfo->name; reginfo++) {
        ixgbe_regdump(hw, reginfo);
    }

    /* Print TX Ring Summary */
    if (!netdev || !netif_running(netdev))
        goto exit;

    dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
    pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
    for (n = 0; n < adapter->num_tx_queues; n++) {
        tx_ring = adapter->tx_ring[n];
        tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
        pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
               n, tx_ring->next_to_use, tx_ring->next_to_clean,
               (u64)dma_unmap_addr(tx_buffer, dma),
               dma_unmap_len(tx_buffer, len),
               tx_buffer->next_to_watch,
               (u64)tx_buffer->time_stamp);
    }

    /* Print TX Rings */
    if (!netif_msg_tx_done(adapter))
        goto rx_ring_summary;

    dev_info(&adapter->pdev->dev, "TX Rings Dump\n");

    /* Transmit Descriptor Formats
     *
     * Advanced Transmit Descriptor
     *   +--------------------------------------------------------------+
     * 0 |         Buffer Address [63:0]                                |
     *   +--------------------------------------------------------------+
     * 8 |  PAYLEN  | PORTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
     *   +--------------------------------------------------------------+
     *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
     */

    for (n = 0; n < adapter->num_tx_queues; n++) {
        tx_ring = adapter->tx_ring[n];
        pr_info("------------------------------------\n");
        pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
        pr_info("------------------------------------\n");
        pr_info("T [desc]     [address 63:0  ] "
            "[PlPOIdStDDt Ln] [bi->dma       ] "
            "leng  ntw timestamp        bi->skb\n");

        for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
            tx_desc = IXGBE_TX_DESC(tx_ring, i);
            tx_buffer = &tx_ring->tx_buffer_info[i];
            u0 = (struct my_u0 *)tx_desc;
            pr_info("T [0x%03X]    %016llX %016llX %016llX"
                " %04X  %p %016llX %p", i,
                le64_to_cpu(u0->a),
                le64_to_cpu(u0->b),
                (u64)dma_unmap_addr(tx_buffer, dma),
                dma_unmap_len(tx_buffer, len),
                tx_buffer->next_to_watch,
                (u64)tx_buffer->time_stamp,
                tx_buffer->skb);
            if (i == tx_ring->next_to_use &&
                i == tx_ring->next_to_clean)
                pr_cont(" NTC/U\n");
            else if (i == tx_ring->next_to_use)
                pr_cont(" NTU\n");
            else if (i == tx_ring->next_to_clean)
                pr_cont(" NTC\n");
            else
                pr_cont("\n");

            if (netif_msg_pktdata(adapter) &&
                tx_buffer->skb)
                print_hex_dump(KERN_INFO, "",
                    DUMP_PREFIX_ADDRESS, 16, 1,
                    tx_buffer->skb->data,
                    dma_unmap_len(tx_buffer, len),
                    true);
        }
    }

    /* Print RX Rings Summary */
rx_ring_summary:
    dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
    pr_info("Queue [NTU] [NTC]\n");
    for (n = 0; n < adapter->num_rx_queues; n++) {
        rx_ring = adapter->rx_ring[n];
        pr_info("%5d %5X %5X\n",
            n, rx_ring->next_to_use, rx_ring->next_to_clean);
    }

    /* Print RX Rings */
    if (!netif_msg_rx_status(adapter))
        goto exit;

    dev_info(&adapter->pdev->dev, "RX Rings Dump\n");

    /* Advanced Receive Descriptor (Read) Format
     *    63                                           1        0
     *    +-----------------------------------------------------+
     *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
     *    +----------------------------------------------+------+
     *  8 |       Header Buffer Address [63:1]           |  DD  |
     *    +-----------------------------------------------------+
     *
     *
     * Advanced Receive Descriptor (Write-Back) Format
     *
     *   63       48 47    32 31  30      21 20 16 15   4 3     0
     *   +------------------------------------------------------+
     * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
     *   | Checksum   Ident  |   |           |    | Type | Type |
     *   +------------------------------------------------------+
     * 8 | VLAN Tag | Length | Extended Error | Extended Status |
     *   +------------------------------------------------------+
     *   63       48 47    32 31            20 19               0
     */
    for (n = 0; n < adapter->num_rx_queues; n++) {
        rx_ring = adapter->rx_ring[n];
        pr_info("------------------------------------\n");
        pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
        pr_info("------------------------------------\n");
        pr_info("R  [desc]      [ PktBuf     A0] "
            "[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
            "<-- Adv Rx Read format\n");
        pr_info("RWB[desc]      [PcsmIpSHl PtRs] "
            "[vl er S cks ln] ---------------- [bi->skb] "
            "<-- Adv Rx Write-Back format\n");

        for (i = 0; i < rx_ring->count; i++) {
            rx_buffer_info = &rx_ring->rx_buffer_info[i];
            rx_desc = IXGBE_RX_DESC(rx_ring, i);
            u0 = (struct my_u0 *)rx_desc;
            staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
            if (staterr & IXGBE_RXD_STAT_DD) {
                /* Descriptor Done */
                pr_info("RWB[0x%03X]     %016llX "
                    "%016llX ---------------- %p", i,
                    le64_to_cpu(u0->a),
                    le64_to_cpu(u0->b),
                    rx_buffer_info->skb);
            } else {
                pr_info("R  [0x%03X]     %016llX "
                    "%016llX %016llX %p", i,
                    le64_to_cpu(u0->a),
                    le64_to_cpu(u0->b),
                    (u64)rx_buffer_info->dma,
                    rx_buffer_info->skb);

                if (netif_msg_pktdata(adapter) &&
                    rx_buffer_info->dma) {
                    print_hex_dump(KERN_INFO, "",
                       DUMP_PREFIX_ADDRESS, 16, 1,
                       page_address(rx_buffer_info->page) +
                            rx_buffer_info->page_offset,
                       ixgbe_rx_bufsz(rx_ring), true);
                }
            }

            if (i == rx_ring->next_to_use)
                pr_cont(" NTU\n");
            else if (i == rx_ring->next_to_clean)
                pr_cont(" NTC\n");
            else
                pr_cont("\n");

        }
    }

exit:
    return;
}

static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
{
    u32 ctrl_ext;

    /* Let firmware take over control of h/w */
    ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
    IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
            ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
}

static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
{
    u32 ctrl_ext;

    /* Let firmware know the driver has taken over */
    ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
    IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
            ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
}

static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
               u8 queue, u8 msix_vector)
{
    u32 ivar, index;
    struct ixgbe_hw *hw = &adapter->hw;
    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        msix_vector |= IXGBE_IVAR_ALLOC_VAL;
        if (direction == -1)
            direction = 0;
        index = (((direction * 64) + queue) >> 2) & 0x1F;
        ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
        ivar &= ~(0xFF << (8 * (queue & 0x3)));
        ivar |= (msix_vector << (8 * (queue & 0x3)));
        IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        if (direction == -1) {
            /* other causes */
            msix_vector |= IXGBE_IVAR_ALLOC_VAL;
            index = ((queue & 1) * 8);
            ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
            ivar &= ~(0xFF << index);
            ivar |= (msix_vector << index);
            IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
            break;
        } else {
            /* tx or rx causes */
            msix_vector |= IXGBE_IVAR_ALLOC_VAL;
            index = ((16 * (queue & 1)) + (8 * direction));
            ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
            ivar &= ~(0xFF << index);
            ivar |= (msix_vector << index);
            IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
            break;
        }
    default:
        break;
    }
}

static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
                      u64 qmask)
{
    u32 mask;

    switch (adapter->hw.mac.type) {
    case ixgbe_mac_82598EB:
        mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        mask = (qmask & 0xFFFFFFFF);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
        mask = (qmask >> 32);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
        break;
    default:
        break;
    }
}

void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring,
                      struct ixgbe_tx_buffer *tx_buffer)
{
    if (tx_buffer->skb) {
        dev_kfree_skb_any(tx_buffer->skb);
        if (dma_unmap_len(tx_buffer, len))
            dma_unmap_single(ring->dev,
                     dma_unmap_addr(tx_buffer, dma),
                     dma_unmap_len(tx_buffer, len),
                     DMA_TO_DEVICE);
    } else if (dma_unmap_len(tx_buffer, len)) {
        dma_unmap_page(ring->dev,
                   dma_unmap_addr(tx_buffer, dma),
                   dma_unmap_len(tx_buffer, len),
                   DMA_TO_DEVICE);
    }
    tx_buffer->next_to_watch = NULL;
    tx_buffer->skb = NULL;
    dma_unmap_len_set(tx_buffer, len, 0);
    /* tx_buffer must be completely set up in the transmit path */
}

static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    struct ixgbe_hw_stats *hwstats = &adapter->stats;
    int i;
    u32 data;

    if ((hw->fc.current_mode != ixgbe_fc_full) &&
        (hw->fc.current_mode != ixgbe_fc_rx_pause))
        return;

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
        break;
    default:
        data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
    }
    hwstats->lxoffrxc += data;

    /* refill credits (no tx hang) if we received xoff */
    if (!data)
        return;

    for (i = 0; i < adapter->num_tx_queues; i++)
        clear_bit(__IXGBE_HANG_CHECK_ARMED,
              &adapter->tx_ring[i]->state);
}

static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    struct ixgbe_hw_stats *hwstats = &adapter->stats;
    u32 xoff[8] = {0};
    int i;
    bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

    if (adapter->ixgbe_ieee_pfc)
        pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

    if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
        ixgbe_update_xoff_rx_lfc(adapter);
        return;
    }

    /* update stats for each tc, only valid with PFC enabled */
    for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
            xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
            break;
        default:
            xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
        }
        hwstats->pxoffrxc[i] += xoff[i];
    }

    /* disarm tx queues that have received xoff frames */
    for (i = 0; i < adapter->num_tx_queues; i++) {
        struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
        u8 tc = tx_ring->dcb_tc;

        if (xoff[tc])
            clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
    }
}

static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
{
    return ring->stats.packets;
}

static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
{
    struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
    struct ixgbe_hw *hw = &adapter->hw;

    u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
    u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));

    if (head != tail)
        return (head < tail) ?
            tail - head : (tail + ring->count - head);

    return 0;
}

static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
{
    u32 tx_done = ixgbe_get_tx_completed(tx_ring);
    u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
    u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
    bool ret = false;

    clear_check_for_tx_hang(tx_ring);

    /*
     * Check for a hung queue, but be thorough. This verifies
     * that a transmit has been completed since the previous
     * check AND there is at least one packet pending. The
     * ARMED bit is set to indicate a potential hang. The
     * bit is cleared if a pause frame is received to remove
     * false hang detection due to PFC or 802.3x frames. By
     * requiring this to fail twice we avoid races with
     * pfc clearing the ARMED bit and conditions where we
     * run the check_tx_hang logic with a transmit completion
     * pending but without time to complete it yet.
     */
    if ((tx_done_old == tx_done) && tx_pending) {
        /* make sure it is true for two checks in a row */
        ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
                       &tx_ring->state);
    } else {
        /* update completed stats and continue */
        tx_ring->tx_stats.tx_done_old = tx_done;
        /* reset the countdown */
        clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
    }

    return ret;
}

static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
{

    /* Do the reset outside of interrupt context */
    if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
        adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
        ixgbe_service_event_schedule(adapter);
    }
}

static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
                   struct ixgbe_ring *tx_ring)
{
    struct ixgbe_adapter *adapter = q_vector->adapter;
    struct ixgbe_tx_buffer *tx_buffer;
    union ixgbe_adv_tx_desc *tx_desc;
    unsigned int total_bytes = 0, total_packets = 0;
    unsigned int budget = q_vector->tx.work_limit;
    unsigned int i = tx_ring->next_to_clean;

    if (test_bit(__IXGBE_DOWN, &adapter->state))
        return true;

    tx_buffer = &tx_ring->tx_buffer_info[i];
    tx_desc = IXGBE_TX_DESC(tx_ring, i);
    i -= tx_ring->count;

    do {
        union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;

        /* if next_to_watch is not set then there is no work pending */
        if (!eop_desc)
            break;

        /* prevent any other reads prior to eop_desc */
        rmb();

        /* if DD is not set pending work has not been completed */
        if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
            break;

        /* clear next_to_watch to prevent false hangs */
        tx_buffer->next_to_watch = NULL;

        /* update the statistics for this packet */
        total_bytes += tx_buffer->bytecount;
        total_packets += tx_buffer->gso_segs;

#ifdef CONFIG_IXGBE_PTP
        if (unlikely(tx_buffer->tx_flags & IXGBE_TX_FLAGS_TSTAMP))
            ixgbe_ptp_tx_hwtstamp(q_vector, tx_buffer->skb);
#endif

        /* free the skb */
        dev_kfree_skb_any(tx_buffer->skb);

        /* unmap skb header data */
        dma_unmap_single(tx_ring->dev,
                 dma_unmap_addr(tx_buffer, dma),
                 dma_unmap_len(tx_buffer, len),
                 DMA_TO_DEVICE);

        /* clear tx_buffer data */
        tx_buffer->skb = NULL;
        dma_unmap_len_set(tx_buffer, len, 0);

        /* unmap remaining buffers */
        while (tx_desc != eop_desc) {
            tx_buffer++;
            tx_desc++;
            i++;
            if (unlikely(!i)) {
                i -= tx_ring->count;
                tx_buffer = tx_ring->tx_buffer_info;
                tx_desc = IXGBE_TX_DESC(tx_ring, 0);
            }

            /* unmap any remaining paged data */
            if (dma_unmap_len(tx_buffer, len)) {
                dma_unmap_page(tx_ring->dev,
                           dma_unmap_addr(tx_buffer, dma),
                           dma_unmap_len(tx_buffer, len),
                           DMA_TO_DEVICE);
                dma_unmap_len_set(tx_buffer, len, 0);
            }
        }

        /* move us one more past the eop_desc for start of next pkt */
        tx_buffer++;
        tx_desc++;
        i++;
        if (unlikely(!i)) {
            i -= tx_ring->count;
            tx_buffer = tx_ring->tx_buffer_info;
            tx_desc = IXGBE_TX_DESC(tx_ring, 0);
        }

        /* issue prefetch for next Tx descriptor */
        prefetch(tx_desc);

        /* update budget accounting */
        budget--;
    } while (likely(budget));

    i += tx_ring->count;
    tx_ring->next_to_clean = i;
    u64_stats_update_begin(&tx_ring->syncp);
    tx_ring->stats.bytes += total_bytes;
    tx_ring->stats.packets += total_packets;
    u64_stats_update_end(&tx_ring->syncp);
    q_vector->tx.total_bytes += total_bytes;
    q_vector->tx.total_packets += total_packets;

    if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
        /* schedule immediate reset if we believe we hung */
        struct ixgbe_hw *hw = &adapter->hw;
        e_err(drv, "Detected Tx Unit Hang\n"
            "  Tx Queue             <%d>\n"
            "  TDH, TDT             <%x>, <%x>\n"
            "  next_to_use          <%x>\n"
            "  next_to_clean        <%x>\n"
            "tx_buffer_info[next_to_clean]\n"
            "  time_stamp           <%lx>\n"
            "  jiffies              <%lx>\n",
            tx_ring->queue_index,
            IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
            IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
            tx_ring->next_to_use, i,
            tx_ring->tx_buffer_info[i].time_stamp, jiffies);

        netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);

        e_info(probe,
               "tx hang %d detected on queue %d, resetting adapter\n",
            adapter->tx_timeout_count + 1, tx_ring->queue_index);

        /* schedule immediate reset if we believe we hung */
        ixgbe_tx_timeout_reset(adapter);

        /* the adapter is about to reset, no point in enabling stuff */
        return true;
    }

    netdev_tx_completed_queue(txring_txq(tx_ring),
                  total_packets, total_bytes);

#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
    if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
             (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
        /* Make sure that anybody stopping the queue after this
         * sees the new next_to_clean.
         */
        smp_mb();
        if (__netif_subqueue_stopped(tx_ring->netdev,
                         tx_ring->queue_index)
            && !test_bit(__IXGBE_DOWN, &adapter->state)) {
            netif_wake_subqueue(tx_ring->netdev,
                        tx_ring->queue_index);
            ++tx_ring->tx_stats.restart_queue;
        }
    }

    return !!budget;
}

#ifdef CONFIG_IXGBE_DCA
static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
                struct ixgbe_ring *tx_ring,
                int cpu)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
    u16 reg_offset;

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
        txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
        break;
    default:
        /* for unknown hardware do not write register */
        return;
    }

    /*
     * We can enable relaxed ordering for reads, but not writes when
     * DCA is enabled.  This is due to a known issue in some chipsets
     * which will cause the DCA tag to be cleared.
     */
    txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
          IXGBE_DCA_TXCTRL_DATA_RRO_EN |
          IXGBE_DCA_TXCTRL_DESC_DCA_EN;

    IXGBE_WRITE_REG(hw, reg_offset, txctrl);
}

static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
                struct ixgbe_ring *rx_ring,
                int cpu)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 rxctrl = dca3_get_tag(rx_ring->dev, cpu);
    u8 reg_idx = rx_ring->reg_idx;


    switch (hw->mac.type) {
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
        break;
    default:
        break;
    }

    /*
     * We can enable relaxed ordering for reads, but not writes when
     * DCA is enabled.  This is due to a known issue in some chipsets
     * which will cause the DCA tag to be cleared.
     */
    rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
          IXGBE_DCA_RXCTRL_DATA_DCA_EN |
          IXGBE_DCA_RXCTRL_DESC_DCA_EN;

    IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
}

static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
{
    struct ixgbe_adapter *adapter = q_vector->adapter;
    struct ixgbe_ring *ring;
    int cpu = get_cpu();

    if (q_vector->cpu == cpu)
        goto out_no_update;

    ixgbe_for_each_ring(ring, q_vector->tx)
        ixgbe_update_tx_dca(adapter, ring, cpu);

    ixgbe_for_each_ring(ring, q_vector->rx)
        ixgbe_update_rx_dca(adapter, ring, cpu);

    q_vector->cpu = cpu;
out_no_update:
    put_cpu();
}

static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
{
    int i;

    if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
        return;

    /* always use CB2 mode, difference is masked in the CB driver */
    IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);

    for (i = 0; i < adapter->num_q_vectors; i++) {
        adapter->q_vector[i]->cpu = -1;
        ixgbe_update_dca(adapter->q_vector[i]);
    }
}

static int __ixgbe_notify_dca(struct device *dev, void *data)
{
    struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
    unsigned long event = *(unsigned long *)data;

    if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
        return 0;

    switch (event) {
    case DCA_PROVIDER_ADD:
        /* if we're already enabled, don't do it again */
        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
            break;
        if (dca_add_requester(dev) == 0) {
            adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
            ixgbe_setup_dca(adapter);
            break;
        }
        /* Fall Through since DCA is disabled. */
    case DCA_PROVIDER_REMOVE:
        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
            dca_remove_requester(dev);
            adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
            IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
        }
        break;
    }

    return 0;
}

#endif /* CONFIG_IXGBE_DCA */
static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
                 union ixgbe_adv_rx_desc *rx_desc,
                 struct sk_buff *skb)
{
    if (ring->netdev->features & NETIF_F_RXHASH)
        skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
}

#ifdef IXGBE_FCOE

static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
                    union ixgbe_adv_rx_desc *rx_desc)
{
    __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;

    return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
           ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
        (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
                 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
}

#endif /* IXGBE_FCOE */

static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
                     union ixgbe_adv_rx_desc *rx_desc,
                     struct sk_buff *skb)
{
    skb_checksum_none_assert(skb);

    /* Rx csum disabled */
    if (!(ring->netdev->features & NETIF_F_RXCSUM))
        return;

    /* if IP and error */
    if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
        ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
        ring->rx_stats.csum_err++;
        return;
    }

    if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
        return;

    if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
        __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;

        /*
         * 82599 errata, UDP frames with a 0 checksum can be marked as
         * checksum errors.
         */
        if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
            test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
            return;

        ring->rx_stats.csum_err++;
        return;
    }

    /* It must be a TCP or UDP packet with a valid checksum */
    skb->ip_summed = CHECKSUM_UNNECESSARY;
}

static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
{
    rx_ring->next_to_use = val;

    /* update next to alloc since we have filled the ring */
    rx_ring->next_to_alloc = val;
    /*
     * Force memory writes to complete before letting h/w
     * know there are new descriptors to fetch.  (Only
     * applicable for weak-ordered memory model archs,
     * such as IA-64).
     */
    wmb();
    writel(val, rx_ring->tail);
}

static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
                    struct ixgbe_rx_buffer *bi)
{
    struct page *page = bi->page;
    dma_addr_t dma = bi->dma;

    /* since we are recycling buffers we should seldom need to alloc */
    if (likely(dma))
        return true;

    /* alloc new page for storage */
    if (likely(!page)) {
        page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP,
                     bi->skb, ixgbe_rx_pg_order(rx_ring));
        if (unlikely(!page)) {
            rx_ring->rx_stats.alloc_rx_page_failed++;
            return false;
        }
        bi->page = page;
    }

    /* map page for use */
    dma = dma_map_page(rx_ring->dev, page, 0,
               ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);

    /*
     * if mapping failed free memory back to system since
     * there isn't much point in holding memory we can't use
     */
    if (dma_mapping_error(rx_ring->dev, dma)) {
        __free_pages(page, ixgbe_rx_pg_order(rx_ring));
        bi->page = NULL;

        rx_ring->rx_stats.alloc_rx_page_failed++;
        return false;
    }

    bi->dma = dma;
    bi->page_offset = 0;

    return true;
}

void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
{
    union ixgbe_adv_rx_desc *rx_desc;
    struct ixgbe_rx_buffer *bi;
    u16 i = rx_ring->next_to_use;

    /* nothing to do */
    if (!cleaned_count)
        return;

    rx_desc = IXGBE_RX_DESC(rx_ring, i);
    bi = &rx_ring->rx_buffer_info[i];
    i -= rx_ring->count;

    do {
        if (!ixgbe_alloc_mapped_page(rx_ring, bi))
            break;

        /*
         * Refresh the desc even if buffer_addrs didn't change
         * because each write-back erases this info.
         */
        rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);

        rx_desc++;
        bi++;
        i++;
        if (unlikely(!i)) {
            rx_desc = IXGBE_RX_DESC(rx_ring, 0);
            bi = rx_ring->rx_buffer_info;
            i -= rx_ring->count;
        }

        /* clear the hdr_addr for the next_to_use descriptor */
        rx_desc->read.hdr_addr = 0;

        cleaned_count--;
    } while (cleaned_count);

    i += rx_ring->count;

    if (rx_ring->next_to_use != i)
        ixgbe_release_rx_desc(rx_ring, i);
}

static unsigned int ixgbe_get_headlen(unsigned char *data,
                      unsigned int max_len)
{
    union {
        unsigned char *network;
        /* l2 headers */
        struct ethhdr *eth;
        struct vlan_hdr *vlan;
        /* l3 headers */
        struct iphdr *ipv4;
    } hdr;
    __be16 protocol;
    u8 nexthdr = 0; /* default to not TCP */
    u8 hlen;

    /* this should never happen, but better safe than sorry */
    if (max_len < ETH_HLEN)
        return max_len;

    /* initialize network frame pointer */
    hdr.network = data;

    /* set first protocol and move network header forward */
    protocol = hdr.eth->h_proto;
    hdr.network += ETH_HLEN;

    /* handle any vlan tag if present */
    if (protocol == __constant_htons(ETH_P_8021Q)) {
        if ((hdr.network - data) > (max_len - VLAN_HLEN))
            return max_len;

        protocol = hdr.vlan->h_vlan_encapsulated_proto;
        hdr.network += VLAN_HLEN;
    }

    /* handle L3 protocols */
    if (protocol == __constant_htons(ETH_P_IP)) {
        if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
            return max_len;

        /* access ihl as a u8 to avoid unaligned access on ia64 */
        hlen = (hdr.network[0] & 0x0F) << 2;

        /* verify hlen meets minimum size requirements */
        if (hlen < sizeof(struct iphdr))
            return hdr.network - data;

        /* record next protocol */
        nexthdr = hdr.ipv4->protocol;
        hdr.network += hlen;
#ifdef IXGBE_FCOE
    } else if (protocol == __constant_htons(ETH_P_FCOE)) {
        if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN))
            return max_len;
        hdr.network += FCOE_HEADER_LEN;
#endif
    } else {
        return hdr.network - data;
    }

    /* finally sort out TCP */
    if (nexthdr == IPPROTO_TCP) {
        if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
            return max_len;

        /* access doff as a u8 to avoid unaligned access on ia64 */
        hlen = (hdr.network[12] & 0xF0) >> 2;

        /* verify hlen meets minimum size requirements */
        if (hlen < sizeof(struct tcphdr))
            return hdr.network - data;

        hdr.network += hlen;
    }

    /*
     * If everything has gone correctly hdr.network should be the
     * data section of the packet and will be the end of the header.
     * If not then it probably represents the end of the last recognized
     * header.
     */
    if ((hdr.network - data) < max_len)
        return hdr.network - data;
    else
        return max_len;
}

static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
                   struct sk_buff *skb)
{
    u16 hdr_len = skb_headlen(skb);

    /* set gso_size to avoid messing up TCP MSS */
    skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
                         IXGBE_CB(skb)->append_cnt);
}

static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
                   struct sk_buff *skb)
{
    /* if append_cnt is 0 then frame is not RSC */
    if (!IXGBE_CB(skb)->append_cnt)
        return;

    rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
    rx_ring->rx_stats.rsc_flush++;

    ixgbe_set_rsc_gso_size(rx_ring, skb);

    /* gso_size is computed using append_cnt so always clear it last */
    IXGBE_CB(skb)->append_cnt = 0;
}

static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
                     union ixgbe_adv_rx_desc *rx_desc,
                     struct sk_buff *skb)
{
    struct net_device *dev = rx_ring->netdev;

    ixgbe_update_rsc_stats(rx_ring, skb);

    ixgbe_rx_hash(rx_ring, rx_desc, skb);

    ixgbe_rx_checksum(rx_ring, rx_desc, skb);

#ifdef CONFIG_IXGBE_PTP
    ixgbe_ptp_rx_hwtstamp(rx_ring->q_vector, rx_desc, skb);
#endif

    if ((dev->features & NETIF_F_HW_VLAN_RX) &&
        ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
        u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
        __vlan_hwaccel_put_tag(skb, vid);
    }

    skb_record_rx_queue(skb, rx_ring->queue_index);

    skb->protocol = eth_type_trans(skb, dev);
}

static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
             struct sk_buff *skb)
{
    struct ixgbe_adapter *adapter = q_vector->adapter;

    if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
        napi_gro_receive(&q_vector->napi, skb);
    else
        netif_rx(skb);
}

static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
                 union ixgbe_adv_rx_desc *rx_desc,
                 struct sk_buff *skb)
{
    u32 ntc = rx_ring->next_to_clean + 1;

    /* fetch, update, and store next to clean */
    ntc = (ntc < rx_ring->count) ? ntc : 0;
    rx_ring->next_to_clean = ntc;

    prefetch(IXGBE_RX_DESC(rx_ring, ntc));

    /* update RSC append count if present */
    if (ring_is_rsc_enabled(rx_ring)) {
        __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
                     cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);

        if (unlikely(rsc_enabled)) {
            u32 rsc_cnt = le32_to_cpu(rsc_enabled);

            rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
            IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;

            /* update ntc based on RSC value */
            ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
            ntc &= IXGBE_RXDADV_NEXTP_MASK;
            ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
        }
    }

    /* if we are the last buffer then there is nothing else to do */
    if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
        return false;

    /* place skb in next buffer to be received */
    rx_ring->rx_buffer_info[ntc].skb = skb;
    rx_ring->rx_stats.non_eop_descs++;

    return true;
}

static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
                struct sk_buff *skb)
{
    struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
    unsigned char *va;
    unsigned int pull_len;

    /*
     * it is valid to use page_address instead of kmap since we are
     * working with pages allocated out of the lomem pool per
     * alloc_page(GFP_ATOMIC)
     */
    va = skb_frag_address(frag);

    /*
     * we need the header to contain the greater of either ETH_HLEN or
     * 60 bytes if the skb->len is less than 60 for skb_pad.
     */
    pull_len = ixgbe_get_headlen(va, IXGBE_RX_HDR_SIZE);

    /* align pull length to size of long to optimize memcpy performance */
    skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));

    /* update all of the pointers */
    skb_frag_size_sub(frag, pull_len);
    frag->page_offset += pull_len;
    skb->data_len -= pull_len;
    skb->tail += pull_len;
}

static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
                struct sk_buff *skb)
{
    /* if the page was released unmap it, else just sync our portion */
    if (unlikely(IXGBE_CB(skb)->page_released)) {
        dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma,
                   ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
        IXGBE_CB(skb)->page_released = false;
    } else {
        struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];

        dma_sync_single_range_for_cpu(rx_ring->dev,
                          IXGBE_CB(skb)->dma,
                          frag->page_offset,
                          ixgbe_rx_bufsz(rx_ring),
                          DMA_FROM_DEVICE);
    }
    IXGBE_CB(skb)->dma = 0;
}

static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
                  union ixgbe_adv_rx_desc *rx_desc,
                  struct sk_buff *skb)
{
    struct net_device *netdev = rx_ring->netdev;

    /* verify that the packet does not have any known errors */
    if (unlikely(ixgbe_test_staterr(rx_desc,
                    IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
        !(netdev->features & NETIF_F_RXALL))) {
        dev_kfree_skb_any(skb);
        return true;
    }

    /* place header in linear portion of buffer */
    if (skb_is_nonlinear(skb))
        ixgbe_pull_tail(rx_ring, skb);

#ifdef IXGBE_FCOE
    /* do not attempt to pad FCoE Frames as this will disrupt DDP */
    if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
        return false;

#endif
    /* if skb_pad returns an error the skb was freed */
    if (unlikely(skb->len < 60)) {
        int pad_len = 60 - skb->len;

        if (skb_pad(skb, pad_len))
            return true;
        __skb_put(skb, pad_len);
    }

    return false;
}

static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
                struct ixgbe_rx_buffer *old_buff)
{
    struct ixgbe_rx_buffer *new_buff;
    u16 nta = rx_ring->next_to_alloc;

    new_buff = &rx_ring->rx_buffer_info[nta];

    /* update, and store next to alloc */
    nta++;
    rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;

    /* transfer page from old buffer to new buffer */
    new_buff->page = old_buff->page;
    new_buff->dma = old_buff->dma;
    new_buff->page_offset = old_buff->page_offset;

    /* sync the buffer for use by the device */
    dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
                     new_buff->page_offset,
                     ixgbe_rx_bufsz(rx_ring),
                     DMA_FROM_DEVICE);
}

static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
                  struct ixgbe_rx_buffer *rx_buffer,
                  union ixgbe_adv_rx_desc *rx_desc,
                  struct sk_buff *skb)
{
    struct page *page = rx_buffer->page;
    unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
#if (PAGE_SIZE < 8192)
    unsigned int truesize = ixgbe_rx_bufsz(rx_ring);
#else
    unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
    unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) -
                   ixgbe_rx_bufsz(rx_ring);
#endif

    if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
        unsigned char *va = page_address(page) + rx_buffer->page_offset;

        memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

        /* we can reuse buffer as-is, just make sure it is local */
        if (likely(page_to_nid(page) == numa_node_id()))
            return true;

        /* this page cannot be reused so discard it */
        put_page(page);
        return false;
    }

    skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
            rx_buffer->page_offset, size, truesize);

    /* avoid re-using remote pages */
    if (unlikely(page_to_nid(page) != numa_node_id()))
        return false;

#if (PAGE_SIZE < 8192)
    /* if we are only owner of page we can reuse it */
    if (unlikely(page_count(page) != 1))
        return false;

    /* flip page offset to other buffer */
    rx_buffer->page_offset ^= truesize;

    /*
     * since we are the only owner of the page and we need to
     * increment it, just set the value to 2 in order to avoid
     * an unecessary locked operation
     */
    atomic_set(&page->_count, 2);
#else
    /* move offset up to the next cache line */
    rx_buffer->page_offset += truesize;

    if (rx_buffer->page_offset > last_offset)
        return false;

    /* bump ref count on page before it is given to the stack */
    get_page(page);
#endif

    return true;
}

static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring,
                         union ixgbe_adv_rx_desc *rx_desc)
{
    struct ixgbe_rx_buffer *rx_buffer;
    struct sk_buff *skb;
    struct page *page;

    rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
    page = rx_buffer->page;
    prefetchw(page);

    skb = rx_buffer->skb;

    if (likely(!skb)) {
        void *page_addr = page_address(page) +
                  rx_buffer->page_offset;

        /* prefetch first cache line of first page */
        prefetch(page_addr);
#if L1_CACHE_BYTES < 128
        prefetch(page_addr + L1_CACHE_BYTES);
#endif

        /* allocate a skb to store the frags */
        skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
                        IXGBE_RX_HDR_SIZE);
        if (unlikely(!skb)) {
            rx_ring->rx_stats.alloc_rx_buff_failed++;
            return NULL;
        }

        /*
         * we will be copying header into skb->data in
         * pskb_may_pull so it is in our interest to prefetch
         * it now to avoid a possible cache miss
         */
        prefetchw(skb->data);

        /*
         * Delay unmapping of the first packet. It carries the
         * header information, HW may still access the header
         * after the writeback.  Only unmap it when EOP is
         * reached
         */
        if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
            goto dma_sync;

        IXGBE_CB(skb)->dma = rx_buffer->dma;
    } else {
        if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
            ixgbe_dma_sync_frag(rx_ring, skb);

dma_sync:
        /* we are reusing so sync this buffer for CPU use */
        dma_sync_single_range_for_cpu(rx_ring->dev,
                          rx_buffer->dma,
                          rx_buffer->page_offset,
                          ixgbe_rx_bufsz(rx_ring),
                          DMA_FROM_DEVICE);
    }

    /* pull page into skb */
    if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
        /* hand second half of page back to the ring */
        ixgbe_reuse_rx_page(rx_ring, rx_buffer);
    } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) {
        /* the page has been released from the ring */
        IXGBE_CB(skb)->page_released = true;
    } else {
        /* we are not reusing the buffer so unmap it */
        dma_unmap_page(rx_ring->dev, rx_buffer->dma,
                   ixgbe_rx_pg_size(rx_ring),
                   DMA_FROM_DEVICE);
    }

    /* clear contents of buffer_info */
    rx_buffer->skb = NULL;
    rx_buffer->dma = 0;
    rx_buffer->page = NULL;

    return skb;
}

static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
                   struct ixgbe_ring *rx_ring,
                   int budget)
{
    unsigned int total_rx_bytes = 0, total_rx_packets = 0;
#ifdef IXGBE_FCOE
    struct ixgbe_adapter *adapter = q_vector->adapter;
    int ddp_bytes;
    unsigned int mss = 0;
#endif /* IXGBE_FCOE */
    u16 cleaned_count = ixgbe_desc_unused(rx_ring);

    do {
        union ixgbe_adv_rx_desc *rx_desc;
        struct sk_buff *skb;

        /* return some buffers to hardware, one at a time is too slow */
        if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
            ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
            cleaned_count = 0;
        }

        rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);

        if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
            break;

        /*
         * This memory barrier is needed to keep us from reading
         * any other fields out of the rx_desc until we know the
         * RXD_STAT_DD bit is set
         */
        rmb();

        /* retrieve a buffer from the ring */
        skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc);

        /* exit if we failed to retrieve a buffer */
        if (!skb)
            break;

        cleaned_count++;

        /* place incomplete frames back on ring for completion */
        if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
            continue;

        /* verify the packet layout is correct */
        if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
            continue;

        /* probably a little skewed due to removing CRC */
        total_rx_bytes += skb->len;
        total_rx_packets++;

        /* populate checksum, timestamp, VLAN, and protocol */
        ixgbe_process_skb_fields(rx_ring, rx_desc, skb);

#ifdef IXGBE_FCOE
        /* if ddp, not passing to ULD unless for FCP_RSP or error */
        if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
            ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
            /* include DDPed FCoE data */
            if (ddp_bytes > 0) {
                if (!mss) {
                    mss = rx_ring->netdev->mtu -
                        sizeof(struct fcoe_hdr) -
                        sizeof(struct fc_frame_header) -
                        sizeof(struct fcoe_crc_eof);
                    if (mss > 512)
                        mss &= ~511;
                }
                total_rx_bytes += ddp_bytes;
                total_rx_packets += DIV_ROUND_UP(ddp_bytes,
                                 mss);
            }
            if (!ddp_bytes) {
                dev_kfree_skb_any(skb);
                continue;
            }
        }

#endif /* IXGBE_FCOE */
        ixgbe_rx_skb(q_vector, skb);

        /* update budget accounting */
        budget--;
    } while (likely(budget));

    u64_stats_update_begin(&rx_ring->syncp);
    rx_ring->stats.packets += total_rx_packets;
    rx_ring->stats.bytes += total_rx_bytes;
    u64_stats_update_end(&rx_ring->syncp);
    q_vector->rx.total_packets += total_rx_packets;
    q_vector->rx.total_bytes += total_rx_bytes;

    if (cleaned_count)
        ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);

    return !!budget;
}

static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
{
    struct ixgbe_q_vector *q_vector;
    int v_idx;
    u32 mask;

    /* Populate MSIX to EITR Select */
    if (adapter->num_vfs > 32) {
        u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
    }

    /*
     * Populate the IVAR table and set the ITR values to the
     * corresponding register.
     */
    for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
        struct ixgbe_ring *ring;
        q_vector = adapter->q_vector[v_idx];

        ixgbe_for_each_ring(ring, q_vector->rx)
            ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);

        ixgbe_for_each_ring(ring, q_vector->tx)
            ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);

        if (q_vector->tx.ring && !q_vector->rx.ring) {
            /* tx only vector */
            if (adapter->tx_itr_setting == 1)
                q_vector->itr = IXGBE_10K_ITR;
            else
                q_vector->itr = adapter->tx_itr_setting;
        } else {
            /* rx or rx/tx vector */
            if (adapter->rx_itr_setting == 1)
                q_vector->itr = IXGBE_20K_ITR;
            else
                q_vector->itr = adapter->rx_itr_setting;
        }

        ixgbe_write_eitr(q_vector);
    }

    switch (adapter->hw.mac.type) {
    case ixgbe_mac_82598EB:
        ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
                   v_idx);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        ixgbe_set_ivar(adapter, -1, 1, v_idx);
        break;
    default:
        break;
    }
    IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);

    /* set up to autoclear timer, and the vectors */
    mask = IXGBE_EIMS_ENABLE_MASK;
    mask &= ~(IXGBE_EIMS_OTHER |
          IXGBE_EIMS_MAILBOX |
          IXGBE_EIMS_LSC);

    IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
}

enum latency_range {
    lowest_latency = 0,
    low_latency = 1,
    bulk_latency = 2,
    latency_invalid = 255
};

static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
                 struct ixgbe_ring_container *ring_container)
{
    int bytes = ring_container->total_bytes;
    int packets = ring_container->total_packets;
    u32 timepassed_us;
    u64 bytes_perint;
    u8 itr_setting = ring_container->itr;

    if (packets == 0)
        return;

    /* simple throttlerate management
     *   0-10MB/s   lowest (100000 ints/s)
     *  10-20MB/s   low    (20000 ints/s)
     *  20-1249MB/s bulk   (8000 ints/s)
     */
    /* what was last interrupt timeslice? */
    timepassed_us = q_vector->itr >> 2;
    bytes_perint = bytes / timepassed_us; /* bytes/usec */

    switch (itr_setting) {
    case lowest_latency:
        if (bytes_perint > 10)
            itr_setting = low_latency;
        break;
    case low_latency:
        if (bytes_perint > 20)
            itr_setting = bulk_latency;
        else if (bytes_perint <= 10)
            itr_setting = lowest_latency;
        break;
    case bulk_latency:
        if (bytes_perint <= 20)
            itr_setting = low_latency;
        break;
    }

    /* clear work counters since we have the values we need */
    ring_container->total_bytes = 0;
    ring_container->total_packets = 0;

    /* write updated itr to ring container */
    ring_container->itr = itr_setting;
}

void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
{
    struct ixgbe_adapter *adapter = q_vector->adapter;
    struct ixgbe_hw *hw = &adapter->hw;
    int v_idx = q_vector->v_idx;
    u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;

    switch (adapter->hw.mac.type) {
    case ixgbe_mac_82598EB:
        /* must write high and low 16 bits to reset counter */
        itr_reg |= (itr_reg << 16);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        /*
         * set the WDIS bit to not clear the timer bits and cause an
         * immediate assertion of the interrupt
         */
        itr_reg |= IXGBE_EITR_CNT_WDIS;
        break;
    default:
        break;
    }
    IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
}

static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
{
    u32 new_itr = q_vector->itr;
    u8 current_itr;

    ixgbe_update_itr(q_vector, &q_vector->tx);
    ixgbe_update_itr(q_vector, &q_vector->rx);

    current_itr = max(q_vector->rx.itr, q_vector->tx.itr);

    switch (current_itr) {
    /* counts and packets in update_itr are dependent on these numbers */
    case lowest_latency:
        new_itr = IXGBE_100K_ITR;
        break;
    case low_latency:
        new_itr = IXGBE_20K_ITR;
        break;
    case bulk_latency:
        new_itr = IXGBE_8K_ITR;
        break;
    default:
        break;
    }

    if (new_itr != q_vector->itr) {
        /* do an exponential smoothing */
        new_itr = (10 * new_itr * q_vector->itr) /
              ((9 * new_itr) + q_vector->itr);

        /* save the algorithm value here */
        q_vector->itr = new_itr;

        ixgbe_write_eitr(q_vector);
    }
}

static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 eicr = adapter->interrupt_event;

    if (test_bit(__IXGBE_DOWN, &adapter->state))
        return;

    if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
        !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
        return;

    adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;

    switch (hw->device_id) {
    case IXGBE_DEV_ID_82599_T3_LOM:
        /*
         * Since the warning interrupt is for both ports
         * we don't have to check if:
         *  - This interrupt wasn't for our port.
         *  - We may have missed the interrupt so always have to
         *    check if we  got a LSC
         */
        if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
            !(eicr & IXGBE_EICR_LSC))
            return;

        if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
            u32 autoneg;
            bool link_up = false;

            hw->mac.ops.check_link(hw, &autoneg, &link_up, false);

            if (link_up)
                return;
        }

        /* Check if this is not due to overtemp */
        if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
            return;

        break;
    default:
        if (!(eicr & IXGBE_EICR_GPI_SDP0))
            return;
        break;
    }
    e_crit(drv,
           "Network adapter has been stopped because it has over heated. "
           "Restart the computer. If the problem persists, "
           "power off the system and replace the adapter\n");

    adapter->interrupt_event = 0;
}

static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
{
    struct ixgbe_hw *hw = &adapter->hw;

    if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
        (eicr & IXGBE_EICR_GPI_SDP1)) {
        e_crit(probe, "Fan has stopped, replace the adapter\n");
        /* write to clear the interrupt */
        IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
    }
}

static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
{
    if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
        return;

    switch (adapter->hw.mac.type) {
    case ixgbe_mac_82599EB:
        /*
         * Need to check link state so complete overtemp check
         * on service task
         */
        if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
            (!test_bit(__IXGBE_DOWN, &adapter->state))) {
            adapter->interrupt_event = eicr;
            adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
            ixgbe_service_event_schedule(adapter);
            return;
        }
        return;
    case ixgbe_mac_X540:
        if (!(eicr & IXGBE_EICR_TS))
            return;
        break;
    default:
        return;
    }

    e_crit(drv,
           "Network adapter has been stopped because it has over heated. "
           "Restart the computer. If the problem persists, "
           "power off the system and replace the adapter\n");
}

static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
{
    struct ixgbe_hw *hw = &adapter->hw;

    if (eicr & IXGBE_EICR_GPI_SDP2) {
        /* Clear the interrupt */
        IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
        if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
            adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
            ixgbe_service_event_schedule(adapter);
        }
    }

    if (eicr & IXGBE_EICR_GPI_SDP1) {
        /* Clear the interrupt */
        IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
        if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
            adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
            ixgbe_service_event_schedule(adapter);
        }
    }
}

static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;

    adapter->lsc_int++;
    adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
    adapter->link_check_timeout = jiffies;
    if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
        IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
        IXGBE_WRITE_FLUSH(hw);
        ixgbe_service_event_schedule(adapter);
    }
}

static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
                       u64 qmask)
{
    u32 mask;
    struct ixgbe_hw *hw = &adapter->hw;

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
        IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        mask = (qmask & 0xFFFFFFFF);
        if (mask)
            IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
        mask = (qmask >> 32);
        if (mask)
            IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
        break;
    default:
        break;
    }
    /* skip the flush */
}

static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
                        u64 qmask)
{
    u32 mask;
    struct ixgbe_hw *hw = &adapter->hw;

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
        IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        mask = (qmask & 0xFFFFFFFF);
        if (mask)
            IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
        mask = (qmask >> 32);
        if (mask)
            IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
        break;
    default:
        break;
    }
    /* skip the flush */
}

static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
                    bool flush)
{
    u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);

    /* don't reenable LSC while waiting for link */
    if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
        mask &= ~IXGBE_EIMS_LSC;

    if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82599EB:
            mask |= IXGBE_EIMS_GPI_SDP0;
            break;
        case ixgbe_mac_X540:
            mask |= IXGBE_EIMS_TS;
            break;
        default:
            break;
        }
    if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
        mask |= IXGBE_EIMS_GPI_SDP1;
    switch (adapter->hw.mac.type) {
    case ixgbe_mac_82599EB:
        mask |= IXGBE_EIMS_GPI_SDP1;
        mask |= IXGBE_EIMS_GPI_SDP2;
    case ixgbe_mac_X540:
        mask |= IXGBE_EIMS_ECC;
        mask |= IXGBE_EIMS_MAILBOX;
        break;
    default:
        break;
    }

#ifdef CONFIG_IXGBE_PTP
    if (adapter->hw.mac.type == ixgbe_mac_X540)
        mask |= IXGBE_EIMS_TIMESYNC;
#endif

    if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
        !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
        mask |= IXGBE_EIMS_FLOW_DIR;

    IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
    if (queues)
        ixgbe_irq_enable_queues(adapter, ~0);
    if (flush)
        IXGBE_WRITE_FLUSH(&adapter->hw);
}

static irqreturn_t ixgbe_msix_other(int irq, void *data)
{
    struct ixgbe_adapter *adapter = data;
    struct ixgbe_hw *hw = &adapter->hw;
    u32 eicr;

    /*
     * Workaround for Silicon errata.  Use clear-by-write instead
     * of clear-by-read.  Reading with EICS will return the
     * interrupt causes without clearing, which later be done
     * with the write to EICR.
     */
    eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
    IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);

    if (eicr & IXGBE_EICR_LSC)
        ixgbe_check_lsc(adapter);

    if (eicr & IXGBE_EICR_MAILBOX)
        ixgbe_msg_task(adapter);

    switch (hw->mac.type) {
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        if (eicr & IXGBE_EICR_ECC)
            e_info(link, "Received unrecoverable ECC Err, please "
                   "reboot\n");
        /* Handle Flow Director Full threshold interrupt */
        if (eicr & IXGBE_EICR_FLOW_DIR) {
            int reinit_count = 0;
            int i;
            for (i = 0; i < adapter->num_tx_queues; i++) {
                struct ixgbe_ring *ring = adapter->tx_ring[i];
                if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
                               &ring->state))
                    reinit_count++;
            }
            if (reinit_count) {
                /* no more flow director interrupts until after init */
                IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
                adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
                ixgbe_service_event_schedule(adapter);
            }
        }
        ixgbe_check_sfp_event(adapter, eicr);
        ixgbe_check_overtemp_event(adapter, eicr);
        break;
    default:
        break;
    }

    ixgbe_check_fan_failure(adapter, eicr);

#ifdef CONFIG_IXGBE_PTP
    if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
        ixgbe_ptp_check_pps_event(adapter, eicr);
#endif

    /* re-enable the original interrupt state, no lsc, no queues */
    if (!test_bit(__IXGBE_DOWN, &adapter->state))
        ixgbe_irq_enable(adapter, false, false);

    return IRQ_HANDLED;
}

static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
{
    struct ixgbe_q_vector *q_vector = data;

    /* EIAM disabled interrupts (on this vector) for us */

    if (q_vector->rx.ring || q_vector->tx.ring)
        napi_schedule(&q_vector->napi);

    return IRQ_HANDLED;
}

int ixgbe_poll(struct napi_struct *napi, int budget)
{
    struct ixgbe_q_vector *q_vector =
                container_of(napi, struct ixgbe_q_vector, napi);
    struct ixgbe_adapter *adapter = q_vector->adapter;
    struct ixgbe_ring *ring;
    int per_ring_budget;
    bool clean_complete = true;

#ifdef CONFIG_IXGBE_DCA
    if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
        ixgbe_update_dca(q_vector);
#endif

    ixgbe_for_each_ring(ring, q_vector->tx)
        clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring);

    /* attempt to distribute budget to each queue fairly, but don't allow
     * the budget to go below 1 because we'll exit polling */
    if (q_vector->rx.count > 1)
        per_ring_budget = max(budget/q_vector->rx.count, 1);
    else
        per_ring_budget = budget;

    ixgbe_for_each_ring(ring, q_vector->rx)
        clean_complete &= ixgbe_clean_rx_irq(q_vector, ring,
                             per_ring_budget);

    /* If all work not completed, return budget and keep polling */
    if (!clean_complete)
        return budget;

    /* all work done, exit the polling mode */
    napi_complete(napi);
    if (adapter->rx_itr_setting & 1)
        ixgbe_set_itr(q_vector);
    if (!test_bit(__IXGBE_DOWN, &adapter->state))
        ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx));

    return 0;
}

static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
{
    struct net_device *netdev = adapter->netdev;
    int vector, err;
    int ri = 0, ti = 0;

    for (vector = 0; vector < adapter->num_q_vectors; vector++) {
        struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
        struct msix_entry *entry = &adapter->msix_entries[vector];

        if (q_vector->tx.ring && q_vector->rx.ring) {
            snprintf(q_vector->name, sizeof(q_vector->name) - 1,
                 "%s-%s-%d", netdev->name, "TxRx", ri++);
            ti++;
        } else if (q_vector->rx.ring) {
            snprintf(q_vector->name, sizeof(q_vector->name) - 1,
                 "%s-%s-%d", netdev->name, "rx", ri++);
        } else if (q_vector->tx.ring) {
            snprintf(q_vector->name, sizeof(q_vector->name) - 1,
                 "%s-%s-%d", netdev->name, "tx", ti++);
        } else {
            /* skip this unused q_vector */
            continue;
        }
        err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
                  q_vector->name, q_vector);
        if (err) {
            e_err(probe, "request_irq failed for MSIX interrupt "
                  "Error: %d\n", err);
            goto free_queue_irqs;
        }
        /* If Flow Director is enabled, set interrupt affinity */
        if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
            /* assign the mask for this irq */
            irq_set_affinity_hint(entry->vector,
                          &q_vector->affinity_mask);
        }
    }

    err = request_irq(adapter->msix_entries[vector].vector,
              ixgbe_msix_other, 0, netdev->name, adapter);
    if (err) {
        e_err(probe, "request_irq for msix_other failed: %d\n", err);
        goto free_queue_irqs;
    }

    return 0;

free_queue_irqs:
    while (vector) {
        vector--;
        irq_set_affinity_hint(adapter->msix_entries[vector].vector,
                      NULL);
        free_irq(adapter->msix_entries[vector].vector,
             adapter->q_vector[vector]);
    }
    adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
    pci_disable_msix(adapter->pdev);
    kfree(adapter->msix_entries);
    adapter->msix_entries = NULL;
    return err;
}

static irqreturn_t ixgbe_intr(int irq, void *data)
{
    struct ixgbe_adapter *adapter = data;
    struct ixgbe_hw *hw = &adapter->hw;
    struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
    u32 eicr;

    /*
     * Workaround for silicon errata #26 on 82598.  Mask the interrupt
     * before the read of EICR.
     */
    IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);

    /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
     * therefore no explicit interrupt disable is necessary */
    eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
    if (!eicr) {
        /*
         * shared interrupt alert!
         * make sure interrupts are enabled because the read will
         * have disabled interrupts due to EIAM
         * finish the workaround of silicon errata on 82598.  Unmask
         * the interrupt that we masked before the EICR read.
         */
        if (!test_bit(__IXGBE_DOWN, &adapter->state))
            ixgbe_irq_enable(adapter, true, true);
        return IRQ_NONE;    /* Not our interrupt */
    }

    if (eicr & IXGBE_EICR_LSC)
        ixgbe_check_lsc(adapter);

    switch (hw->mac.type) {
    case ixgbe_mac_82599EB:
        ixgbe_check_sfp_event(adapter, eicr);
        /* Fall through */
    case ixgbe_mac_X540:
        if (eicr & IXGBE_EICR_ECC)
            e_info(link, "Received unrecoverable ECC err, please "
                     "reboot\n");
        ixgbe_check_overtemp_event(adapter, eicr);
        break;
    default:
        break;
    }

    ixgbe_check_fan_failure(adapter, eicr);
#ifdef CONFIG_IXGBE_PTP
    if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
        ixgbe_ptp_check_pps_event(adapter, eicr);
#endif

    /* would disable interrupts here but EIAM disabled it */
    napi_schedule(&q_vector->napi);

    /*
     * re-enable link(maybe) and non-queue interrupts, no flush.
     * ixgbe_poll will re-enable the queue interrupts
     */
    if (!test_bit(__IXGBE_DOWN, &adapter->state))
        ixgbe_irq_enable(adapter, false, false);

    return IRQ_HANDLED;
}

static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
{
    struct net_device *netdev = adapter->netdev;
    int err;

    if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
        err = ixgbe_request_msix_irqs(adapter);
    else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
        err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
                  netdev->name, adapter);
    else
        err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
                  netdev->name, adapter);

    if (err)
        e_err(probe, "request_irq failed, Error %d\n", err);

    return err;
}

static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
{
    int vector;

    if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
        free_irq(adapter->pdev->irq, adapter);
        return;
    }

    for (vector = 0; vector < adapter->num_q_vectors; vector++) {
        struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
        struct msix_entry *entry = &adapter->msix_entries[vector];

        /* free only the irqs that were actually requested */
        if (!q_vector->rx.ring && !q_vector->tx.ring)
            continue;

        /* clear the affinity_mask in the IRQ descriptor */
        irq_set_affinity_hint(entry->vector, NULL);

        free_irq(entry->vector, q_vector);
    }

    free_irq(adapter->msix_entries[vector++].vector, adapter);
}

static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
{
    switch (adapter->hw.mac.type) {
    case ixgbe_mac_82598EB:
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
        break;
    default:
        break;
    }
    IXGBE_WRITE_FLUSH(&adapter->hw);
    if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
        int vector;

        for (vector = 0; vector < adapter->num_q_vectors; vector++)
            synchronize_irq(adapter->msix_entries[vector].vector);

        synchronize_irq(adapter->msix_entries[vector++].vector);
    } else {
        synchronize_irq(adapter->pdev->irq);
    }
}

static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
{
    struct ixgbe_q_vector *q_vector = adapter->q_vector[0];

    /* rx/tx vector */
    if (adapter->rx_itr_setting == 1)
        q_vector->itr = IXGBE_20K_ITR;
    else
        q_vector->itr = adapter->rx_itr_setting;

    ixgbe_write_eitr(q_vector);

    ixgbe_set_ivar(adapter, 0, 0, 0);
    ixgbe_set_ivar(adapter, 1, 0, 0);

    e_info(hw, "Legacy interrupt IVAR setup done\n");
}

void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
                 struct ixgbe_ring *ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u64 tdba = ring->dma;
    int wait_loop = 10;
    u32 txdctl = IXGBE_TXDCTL_ENABLE;
    u8 reg_idx = ring->reg_idx;

    /* disable queue to avoid issues while updating state */
    IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
    IXGBE_WRITE_FLUSH(hw);

    IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
            (tdba & DMA_BIT_MASK(32)));
    IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
    IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
            ring->count * sizeof(union ixgbe_adv_tx_desc));
    IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
    IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
    ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);

    /*
     * set WTHRESH to encourage burst writeback, it should not be set
     * higher than 1 when ITR is 0 as it could cause false TX hangs
     *
     * In order to avoid issues WTHRESH + PTHRESH should always be equal
     * to or less than the number of on chip descriptors, which is
     * currently 40.
     */
    if (!ring->q_vector || (ring->q_vector->itr < 8))
        txdctl |= (1 << 16);    /* WTHRESH = 1 */
    else
        txdctl |= (8 << 16);    /* WTHRESH = 8 */

    /*
     * Setting PTHRESH to 32 both improves performance
     * and avoids a TX hang with DFP enabled
     */
    txdctl |= (1 << 8) |    /* HTHRESH = 1 */
           32;      /* PTHRESH = 32 */

    /* reinitialize flowdirector state */
    if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
        ring->atr_sample_rate = adapter->atr_sample_rate;
        ring->atr_count = 0;
        set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
    } else {
        ring->atr_sample_rate = 0;
    }

    clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);

    /* enable queue */
    IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);

    /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
    if (hw->mac.type == ixgbe_mac_82598EB &&
        !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
        return;

    /* poll to verify queue is enabled */
    do {
        usleep_range(1000, 2000);
        txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
    } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
    if (!wait_loop)
        e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
}

static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 rttdcs, mtqc;
    u8 tcs = netdev_get_num_tc(adapter->netdev);

    if (hw->mac.type == ixgbe_mac_82598EB)
        return;

    /* disable the arbiter while setting MTQC */
    rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
    rttdcs |= IXGBE_RTTDCS_ARBDIS;
    IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);

    /* set transmit pool layout */
    if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
        mtqc = IXGBE_MTQC_VT_ENA;
        if (tcs > 4)
            mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
        else if (tcs > 1)
            mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
        else if (adapter->ring_feature[RING_F_RSS].indices == 4)
            mtqc |= IXGBE_MTQC_32VF;
        else
            mtqc |= IXGBE_MTQC_64VF;
    } else {
        if (tcs > 4)
            mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
        else if (tcs > 1)
            mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
        else
            mtqc = IXGBE_MTQC_64Q_1PB;
    }

    IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);

    /* Enable Security TX Buffer IFG for multiple pb */
    if (tcs) {
        u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
        sectx |= IXGBE_SECTX_DCB;
        IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
    }

    /* re-enable the arbiter */
    rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
    IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
}

static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 dmatxctl;
    u32 i;

    ixgbe_setup_mtqc(adapter);

    if (hw->mac.type != ixgbe_mac_82598EB) {
        /* DMATXCTL.EN must be before Tx queues are enabled */
        dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
        dmatxctl |= IXGBE_DMATXCTL_TE;
        IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
    }

    /* Setup the HW Tx Head and Tail descriptor pointers */
    for (i = 0; i < adapter->num_tx_queues; i++)
        ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
}

static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
                 struct ixgbe_ring *ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u8 reg_idx = ring->reg_idx;
    u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));

    srrctl |= IXGBE_SRRCTL_DROP_EN;

    IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
                  struct ixgbe_ring *ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u8 reg_idx = ring->reg_idx;
    u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));

    srrctl &= ~IXGBE_SRRCTL_DROP_EN;

    IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

#ifdef CONFIG_IXGBE_DCB
void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
#else
static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
#endif
{
    int i;
    bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

    if (adapter->ixgbe_ieee_pfc)
        pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

    /*
     * We should set the drop enable bit if:
     *  SR-IOV is enabled
     *   or
     *  Number of Rx queues > 1 and flow control is disabled
     *
     *  This allows us to avoid head of line blocking for security
     *  and performance reasons.
     */
    if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
        !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
        for (i = 0; i < adapter->num_rx_queues; i++)
            ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
    } else {
        for (i = 0; i < adapter->num_rx_queues; i++)
            ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
    }
}

#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2

static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
                   struct ixgbe_ring *rx_ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 srrctl;
    u8 reg_idx = rx_ring->reg_idx;

    if (hw->mac.type == ixgbe_mac_82598EB) {
        u16 mask = adapter->ring_feature[RING_F_RSS].mask;

        /*
         * if VMDq is not active we must program one srrctl register
         * per RSS queue since we have enabled RDRXCTL.MVMEN
         */
        reg_idx &= mask;
    }

    /* configure header buffer length, needed for RSC */
    srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;

    /* configure the packet buffer length */
    srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;

    /* configure descriptor type */
    srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;

    IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
              0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
              0x6A3E67EA, 0x14364D17, 0x3BED200D};
    u32 mrqc = 0, reta = 0;
    u32 rxcsum;
    int i, j;
    u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;

    /*
     * Program table for at least 2 queues w/ SR-IOV so that VFs can
     * make full use of any rings they may have.  We will use the
     * PSRTYPE register to control how many rings we use within the PF.
     */
    if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2))
        rss_i = 2;

    /* Fill out hash function seeds */
    for (i = 0; i < 10; i++)
        IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);

    /* Fill out redirection table */
    for (i = 0, j = 0; i < 128; i++, j++) {
        if (j == rss_i)
            j = 0;
        /* reta = 4-byte sliding window of
         * 0x00..(indices-1)(indices-1)00..etc. */
        reta = (reta << 8) | (j * 0x11);
        if ((i & 3) == 3)
            IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
    }

    /* Disable indicating checksum in descriptor, enables RSS hash */
    rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
    rxcsum |= IXGBE_RXCSUM_PCSD;
    IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);

    if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
        if (adapter->ring_feature[RING_F_RSS].mask)
            mrqc = IXGBE_MRQC_RSSEN;
    } else {
        u8 tcs = netdev_get_num_tc(adapter->netdev);

        if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
            if (tcs > 4)
                mrqc = IXGBE_MRQC_VMDQRT8TCEN;  /* 8 TCs */
            else if (tcs > 1)
                mrqc = IXGBE_MRQC_VMDQRT4TCEN;  /* 4 TCs */
            else if (adapter->ring_feature[RING_F_RSS].indices == 4)
                mrqc = IXGBE_MRQC_VMDQRSS32EN;
            else
                mrqc = IXGBE_MRQC_VMDQRSS64EN;
        } else {
            if (tcs > 4)
                mrqc = IXGBE_MRQC_RTRSS8TCEN;
            else if (tcs > 1)
                mrqc = IXGBE_MRQC_RTRSS4TCEN;
            else
                mrqc = IXGBE_MRQC_RSSEN;
        }
    }

    /* Perform hash on these packet types */
    mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4 |
        IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
        IXGBE_MRQC_RSS_FIELD_IPV6 |
        IXGBE_MRQC_RSS_FIELD_IPV6_TCP;

    if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
        mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
    if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
        mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;

    IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
}

static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
                   struct ixgbe_ring *ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 rscctrl;
    u8 reg_idx = ring->reg_idx;

    if (!ring_is_rsc_enabled(ring))
        return;

    rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
    rscctrl |= IXGBE_RSCCTL_RSCEN;
    /*
     * we must limit the number of descriptors so that the
     * total size of max desc * buf_len is not greater
     * than 65536
     */
    rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
    IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
}

#define IXGBE_MAX_RX_DESC_POLL 10
static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
                       struct ixgbe_ring *ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int wait_loop = IXGBE_MAX_RX_DESC_POLL;
    u32 rxdctl;
    u8 reg_idx = ring->reg_idx;

    /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
    if (hw->mac.type == ixgbe_mac_82598EB &&
        !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
        return;

    do {
        usleep_range(1000, 2000);
        rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
    } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));

    if (!wait_loop) {
        e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
              "the polling period\n", reg_idx);
    }
}

void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
                struct ixgbe_ring *ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int wait_loop = IXGBE_MAX_RX_DESC_POLL;
    u32 rxdctl;
    u8 reg_idx = ring->reg_idx;

    rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
    rxdctl &= ~IXGBE_RXDCTL_ENABLE;

    /* write value back with RXDCTL.ENABLE bit cleared */
    IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);

    if (hw->mac.type == ixgbe_mac_82598EB &&
        !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
        return;

    /* the hardware may take up to 100us to really disable the rx queue */
    do {
        udelay(10);
        rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
    } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));

    if (!wait_loop) {
        e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
              "the polling period\n", reg_idx);
    }
}

void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
                 struct ixgbe_ring *ring)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u64 rdba = ring->dma;
    u32 rxdctl;
    u8 reg_idx = ring->reg_idx;

    /* disable queue to avoid issues while updating state */
    rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
    ixgbe_disable_rx_queue(adapter, ring);

    IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
    IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
    IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
            ring->count * sizeof(union ixgbe_adv_rx_desc));
    IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
    IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
    ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);

    ixgbe_configure_srrctl(adapter, ring);
    ixgbe_configure_rscctl(adapter, ring);

    /* If operating in IOV mode set RLPML for X540 */
    if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
        hw->mac.type == ixgbe_mac_X540) {
        rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
        rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
                ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
    }

    if (hw->mac.type == ixgbe_mac_82598EB) {
        /*
         * enable cache line friendly hardware writes:
         * PTHRESH=32 descriptors (half the internal cache),
         * this also removes ugly rx_no_buffer_count increment
         * HTHRESH=4 descriptors (to minimize latency on fetch)
         * WTHRESH=8 burst writeback up to two cache lines
         */
        rxdctl &= ~0x3FFFFF;
        rxdctl |=  0x080420;
    }

    /* enable receive descriptor ring */
    rxdctl |= IXGBE_RXDCTL_ENABLE;
    IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);

    ixgbe_rx_desc_queue_enable(adapter, ring);
    ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
}

static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int rss_i = adapter->ring_feature[RING_F_RSS].indices;
    int p;

    /* PSRTYPE must be initialized in non 82598 adapters */
    u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
              IXGBE_PSRTYPE_UDPHDR |
              IXGBE_PSRTYPE_IPV4HDR |
              IXGBE_PSRTYPE_L2HDR |
              IXGBE_PSRTYPE_IPV6HDR;

    if (hw->mac.type == ixgbe_mac_82598EB)
        return;

    if (rss_i > 3)
        psrtype |= 2 << 29;
    else if (rss_i > 1)
        psrtype |= 1 << 29;

    for (p = 0; p < adapter->num_rx_pools; p++)
        IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(p)),
                psrtype);
}

static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 reg_offset, vf_shift;
    u32 gcr_ext, vmdctl;
    int i;

    if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
        return;

    vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
    vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;
    vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
    vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;
    vmdctl |= IXGBE_VT_CTL_REPLEN;
    IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);

    vf_shift = VMDQ_P(0) % 32;
    reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;

    /* Enable only the PF's pool for Tx/Rx */
    IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (~0) << vf_shift);
    IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);
    IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (~0) << vf_shift);
    IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);
    IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);

    /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
    hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));

    /*
     * Set up VF register offsets for selected VT Mode,
     * i.e. 32 or 64 VFs for SR-IOV
     */
    switch (adapter->ring_feature[RING_F_VMDQ].mask) {
    case IXGBE_82599_VMDQ_8Q_MASK:
        gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;
        break;
    case IXGBE_82599_VMDQ_4Q_MASK:
        gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;
        break;
    default:
        gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;
        break;
    }

    IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);

    /* enable Tx loopback for VF/PF communication */
    IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);

    /* Enable MAC Anti-Spoofing */
    hw->mac.ops.set_mac_anti_spoofing(hw, (adapter->num_vfs != 0),
                      adapter->num_vfs);
    /* For VFs that have spoof checking turned off */
    for (i = 0; i < adapter->num_vfs; i++) {
        if (!adapter->vfinfo[i].spoofchk_enabled)
            ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);
    }
}

static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    struct net_device *netdev = adapter->netdev;
    int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
    struct ixgbe_ring *rx_ring;
    int i;
    u32 mhadd, hlreg0;

#ifdef IXGBE_FCOE
    /* adjust max frame to be able to do baby jumbo for FCoE */
    if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
        (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
        max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;

#endif /* IXGBE_FCOE */
    mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
    if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
        mhadd &= ~IXGBE_MHADD_MFS_MASK;
        mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;

        IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
    }

    /* MHADD will allow an extra 4 bytes past for vlan tagged frames */
    max_frame += VLAN_HLEN;

    hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
    /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
    hlreg0 |= IXGBE_HLREG0_JUMBOEN;
    IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);

    /*
     * Setup the HW Rx Head and Tail Descriptor Pointers and
     * the Base and Length of the Rx Descriptor Ring
     */
    for (i = 0; i < adapter->num_rx_queues; i++) {
        rx_ring = adapter->rx_ring[i];
        if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
            set_ring_rsc_enabled(rx_ring);
        else
            clear_ring_rsc_enabled(rx_ring);
    }
}

static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        /*
         * For VMDq support of different descriptor types or
         * buffer sizes through the use of multiple SRRCTL
         * registers, RDRXCTL.MVMEN must be set to 1
         *
         * also, the manual doesn't mention it clearly but DCA hints
         * will only use queue 0's tags unless this bit is set.  Side
         * effects of setting this bit are only that SRRCTL must be
         * fully programmed [0..15]
         */
        rdrxctl |= IXGBE_RDRXCTL_MVMEN;
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        /* Disable RSC for ACK packets */
        IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
           (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
        rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
        /* hardware requires some bits to be set by default */
        rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
        rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
        break;
    default:
        /* We should do nothing since we don't know this hardware */
        return;
    }

    IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
}

static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int i;
    u32 rxctrl;

    /* disable receives while setting up the descriptors */
    rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
    IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);

    ixgbe_setup_psrtype(adapter);
    ixgbe_setup_rdrxctl(adapter);

    /* Program registers for the distribution of queues */
    ixgbe_setup_mrqc(adapter);

    /* set_rx_buffer_len must be called before ring initialization */
    ixgbe_set_rx_buffer_len(adapter);

    /*
     * Setup the HW Rx Head and Tail Descriptor Pointers and
     * the Base and Length of the Rx Descriptor Ring
     */
    for (i = 0; i < adapter->num_rx_queues; i++)
        ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);

    /* disable drop enable for 82598 parts */
    if (hw->mac.type == ixgbe_mac_82598EB)
        rxctrl |= IXGBE_RXCTRL_DMBYPS;

    /* enable all receives */
    rxctrl |= IXGBE_RXCTRL_RXEN;
    hw->mac.ops.enable_rx_dma(hw, rxctrl);
}

static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);
    struct ixgbe_hw *hw = &adapter->hw;

    /* add VID to filter table */
    hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true);
    set_bit(vid, adapter->active_vlans);

    return 0;
}

static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);
    struct ixgbe_hw *hw = &adapter->hw;

    /* remove VID from filter table */
    hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), false);
    clear_bit(vid, adapter->active_vlans);

    return 0;
}

static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 vlnctrl;

    vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
    vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
    IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
}

static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 vlnctrl;

    vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
    vlnctrl |= IXGBE_VLNCTRL_VFE;
    vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
    IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
}

static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 vlnctrl;
    int i, j;

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
        vlnctrl &= ~IXGBE_VLNCTRL_VME;
        IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        for (i = 0; i < adapter->num_rx_queues; i++) {
            j = adapter->rx_ring[i]->reg_idx;
            vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
            vlnctrl &= ~IXGBE_RXDCTL_VME;
            IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
        }
        break;
    default:
        break;
    }
}

static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 vlnctrl;
    int i, j;

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
        vlnctrl |= IXGBE_VLNCTRL_VME;
        IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        for (i = 0; i < adapter->num_rx_queues; i++) {
            j = adapter->rx_ring[i]->reg_idx;
            vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
            vlnctrl |= IXGBE_RXDCTL_VME;
            IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
        }
        break;
    default:
        break;
    }
}

static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
{
    u16 vid;

    ixgbe_vlan_rx_add_vid(adapter->netdev, 0);

    for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
        ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
}

static int ixgbe_write_uc_addr_list(struct net_device *netdev)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);
    struct ixgbe_hw *hw = &adapter->hw;
    unsigned int rar_entries = hw->mac.num_rar_entries - 1;
    int count = 0;

    /* In SR-IOV mode significantly less RAR entries are available */
    if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
        rar_entries = IXGBE_MAX_PF_MACVLANS - 1;

    /* return ENOMEM indicating insufficient memory for addresses */
    if (netdev_uc_count(netdev) > rar_entries)
        return -ENOMEM;

    if (!netdev_uc_empty(netdev)) {
        struct netdev_hw_addr *ha;
        /* return error if we do not support writing to RAR table */
        if (!hw->mac.ops.set_rar)
            return -ENOMEM;

        netdev_for_each_uc_addr(ha, netdev) {
            if (!rar_entries)
                break;
            hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
                        VMDQ_P(0), IXGBE_RAH_AV);
            count++;
        }
    }
    /* write the addresses in reverse order to avoid write combining */
    for (; rar_entries > 0 ; rar_entries--)
        hw->mac.ops.clear_rar(hw, rar_entries);

    return count;
}

void ixgbe_set_rx_mode(struct net_device *netdev)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);
    struct ixgbe_hw *hw = &adapter->hw;
    u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
    int count;

    /* Check for Promiscuous and All Multicast modes */

    fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);

    /* set all bits that we expect to always be set */
    fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */
    fctrl |= IXGBE_FCTRL_BAM;
    fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
    fctrl |= IXGBE_FCTRL_PMCF;

    /* clear the bits we are changing the status of */
    fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);

    if (netdev->flags & IFF_PROMISC) {
        hw->addr_ctrl.user_set_promisc = true;
        fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
        vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
        /* don't hardware filter vlans in promisc mode */
        ixgbe_vlan_filter_disable(adapter);
    } else {
        if (netdev->flags & IFF_ALLMULTI) {
            fctrl |= IXGBE_FCTRL_MPE;
            vmolr |= IXGBE_VMOLR_MPE;
        } else {
            /*
             * Write addresses to the MTA, if the attempt fails
             * then we should just turn on promiscuous mode so
             * that we can at least receive multicast traffic
             */
            hw->mac.ops.update_mc_addr_list(hw, netdev);
            vmolr |= IXGBE_VMOLR_ROMPE;
        }
        ixgbe_vlan_filter_enable(adapter);
        hw->addr_ctrl.user_set_promisc = false;
    }

    /*
     * Write addresses to available RAR registers, if there is not
     * sufficient space to store all the addresses then enable
     * unicast promiscuous mode
     */
    count = ixgbe_write_uc_addr_list(netdev);
    if (count < 0) {
        fctrl |= IXGBE_FCTRL_UPE;
        vmolr |= IXGBE_VMOLR_ROPE;
    }

    if (adapter->num_vfs)
        ixgbe_restore_vf_multicasts(adapter);

    if (hw->mac.type != ixgbe_mac_82598EB) {
        vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &
             ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
               IXGBE_VMOLR_ROPE);
        IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);
    }

    /* This is useful for sniffing bad packets. */
    if (adapter->netdev->features & NETIF_F_RXALL) {
        /* UPE and MPE will be handled by normal PROMISC logic
         * in e1000e_set_rx_mode */
        fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */
              IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */
              IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */

        fctrl &= ~(IXGBE_FCTRL_DPF);
        /* NOTE:  VLAN filtering is disabled by setting PROMISC */
    }

    IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);

    if (netdev->features & NETIF_F_HW_VLAN_RX)
        ixgbe_vlan_strip_enable(adapter);
    else
        ixgbe_vlan_strip_disable(adapter);
}

static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
{
    int q_idx;

    for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
        napi_enable(&adapter->q_vector[q_idx]->napi);
}

static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
{
    int q_idx;

    for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
        napi_disable(&adapter->q_vector[q_idx]->napi);
}

#ifdef CONFIG_IXGBE_DCB

static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;

    if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
        if (hw->mac.type == ixgbe_mac_82598EB)
            netif_set_gso_max_size(adapter->netdev, 65536);
        return;
    }

    if (hw->mac.type == ixgbe_mac_82598EB)
        netif_set_gso_max_size(adapter->netdev, 32768);

#ifdef IXGBE_FCOE
    if (adapter->netdev->features & NETIF_F_FCOE_MTU)
        max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
#endif

    /* reconfigure the hardware */
    if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
        ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
                        DCB_TX_CONFIG);
        ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
                        DCB_RX_CONFIG);
        ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
    } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
        ixgbe_dcb_hw_ets(&adapter->hw,
                 adapter->ixgbe_ieee_ets,
                 max_frame);
        ixgbe_dcb_hw_pfc_config(&adapter->hw,
                    adapter->ixgbe_ieee_pfc->pfc_en,
                    adapter->ixgbe_ieee_ets->prio_tc);
    }

    /* Enable RSS Hash per TC */
    if (hw->mac.type != ixgbe_mac_82598EB) {
        u32 msb = 0;
        u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;

        while (rss_i) {
            msb++;
            rss_i >>= 1;
        }

        /* write msb to all 8 TCs in one write */
        IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);
    }
}
#endif

/* Additional bittime to account for IXGBE framing */
#define IXGBE_ETH_FRAMING 20

static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
{
    struct ixgbe_hw *hw = &adapter->hw;
    struct net_device *dev = adapter->netdev;
    int link, tc, kb, marker;
    u32 dv_id, rx_pba;

    /* Calculate max LAN frame size */
    tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;

#ifdef IXGBE_FCOE
    /* FCoE traffic class uses FCOE jumbo frames */
    if ((dev->features & NETIF_F_FCOE_MTU) &&
        (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) &&
        (pb == ixgbe_fcoe_get_tc(adapter)))
        tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;

#endif
    /* Calculate delay value for device */
    switch (hw->mac.type) {
    case ixgbe_mac_X540:
        dv_id = IXGBE_DV_X540(link, tc);
        break;
    default:
        dv_id = IXGBE_DV(link, tc);
        break;
    }

    /* Loopback switch introduces additional latency */
    if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
        dv_id += IXGBE_B2BT(tc);

    /* Delay value is calculated in bit times convert to KB */
    kb = IXGBE_BT2KB(dv_id);
    rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;

    marker = rx_pba - kb;

    /* It is possible that the packet buffer is not large enough
     * to provide required headroom. In this case throw an error
     * to user and a do the best we can.
     */
    if (marker < 0) {
        e_warn(drv, "Packet Buffer(%i) can not provide enough"
                "headroom to support flow control."
                "Decrease MTU or number of traffic classes\n", pb);
        marker = tc + 1;
    }

    return marker;
}

static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    struct net_device *dev = adapter->netdev;
    int tc;
    u32 dv_id;

    /* Calculate max LAN frame size */
    tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;

    /* Calculate delay value for device */
    switch (hw->mac.type) {
    case ixgbe_mac_X540:
        dv_id = IXGBE_LOW_DV_X540(tc);
        break;
    default:
        dv_id = IXGBE_LOW_DV(tc);
        break;
    }

    /* Delay value is calculated in bit times convert to KB */
    return IXGBE_BT2KB(dv_id);
}

/*
 * ixgbe_pbthresh_setup - calculate and setup high low water marks
 */
static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int num_tc = netdev_get_num_tc(adapter->netdev);
    int i;

    if (!num_tc)
        num_tc = 1;

    hw->fc.low_water = ixgbe_lpbthresh(adapter);

    for (i = 0; i < num_tc; i++) {
        hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);

        /* Low water marks must not be larger than high water marks */
        if (hw->fc.low_water > hw->fc.high_water[i])
            hw->fc.low_water = 0;
    }
}

static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int hdrm;
    u8 tc = netdev_get_num_tc(adapter->netdev);

    if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
        adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
        hdrm = 32 << adapter->fdir_pballoc;
    else
        hdrm = 0;

    hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
    ixgbe_pbthresh_setup(adapter);
}

static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    struct hlist_node *node, *node2;
    struct ixgbe_fdir_filter *filter;

    spin_lock(&adapter->fdir_perfect_lock);

    if (!hlist_empty(&adapter->fdir_filter_list))
        ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);

    hlist_for_each_entry_safe(filter, node, node2,
                  &adapter->fdir_filter_list, fdir_node) {
        ixgbe_fdir_write_perfect_filter_82599(hw,
                &filter->filter,
                filter->sw_idx,
                (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
                IXGBE_FDIR_DROP_QUEUE :
                adapter->rx_ring[filter->action]->reg_idx);
    }

    spin_unlock(&adapter->fdir_perfect_lock);
}

static void ixgbe_configure(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;

    ixgbe_configure_pb(adapter);
#ifdef CONFIG_IXGBE_DCB
    ixgbe_configure_dcb(adapter);
#endif
    /*
     * We must restore virtualization before VLANs or else
     * the VLVF registers will not be populated
     */
    ixgbe_configure_virtualization(adapter);

    ixgbe_set_rx_mode(adapter->netdev);
    ixgbe_restore_vlan(adapter);

    switch (hw->mac.type) {
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        hw->mac.ops.disable_rx_buff(hw);
        break;
    default:
        break;
    }

    if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
        ixgbe_init_fdir_signature_82599(&adapter->hw,
                        adapter->fdir_pballoc);
    } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
        ixgbe_init_fdir_perfect_82599(&adapter->hw,
                          adapter->fdir_pballoc);
        ixgbe_fdir_filter_restore(adapter);
    }

    switch (hw->mac.type) {
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        hw->mac.ops.enable_rx_buff(hw);
        break;
    default:
        break;
    }

#ifdef IXGBE_FCOE
    /* configure FCoE L2 filters, redirection table, and Rx control */
    ixgbe_configure_fcoe(adapter);

#endif /* IXGBE_FCOE */
    ixgbe_configure_tx(adapter);
    ixgbe_configure_rx(adapter);
}

static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
{
    switch (hw->phy.type) {
    case ixgbe_phy_sfp_avago:
    case ixgbe_phy_sfp_ftl:
    case ixgbe_phy_sfp_intel:
    case ixgbe_phy_sfp_unknown:
    case ixgbe_phy_sfp_passive_tyco:
    case ixgbe_phy_sfp_passive_unknown:
    case ixgbe_phy_sfp_active_unknown:
    case ixgbe_phy_sfp_ftl_active:
        return true;
    case ixgbe_phy_nl:
        if (hw->mac.type == ixgbe_mac_82598EB)
            return true;
    default:
        return false;
    }
}

static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
{
    /*
     * We are assuming the worst case scenario here, and that
     * is that an SFP was inserted/removed after the reset
     * but before SFP detection was enabled.  As such the best
     * solution is to just start searching as soon as we start
     */
    if (adapter->hw.mac.type == ixgbe_mac_82598EB)
        adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;

    adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
}

static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
{
    u32 autoneg;
    bool negotiation, link_up = false;
    u32 ret = IXGBE_ERR_LINK_SETUP;

    if (hw->mac.ops.check_link)
        ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);

    if (ret)
        goto link_cfg_out;

    autoneg = hw->phy.autoneg_advertised;
    if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
        ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
                            &negotiation);
    if (ret)
        goto link_cfg_out;

    if (hw->mac.ops.setup_link)
        ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
link_cfg_out:
    return ret;
}

static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 gpie = 0;

    if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
        gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
               IXGBE_GPIE_OCD;
        gpie |= IXGBE_GPIE_EIAME;
        /*
         * use EIAM to auto-mask when MSI-X interrupt is asserted
         * this saves a register write for every interrupt
         */
        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
            IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
            break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        default:
            IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
            IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
            break;
        }
    } else {
        /* legacy interrupts, use EIAM to auto-mask when reading EICR,
         * specifically only auto mask tx and rx interrupts */
        IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
    }

    /* XXX: to interrupt immediately for EICS writes, enable this */
    /* gpie |= IXGBE_GPIE_EIMEN; */

    if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
        gpie &= ~IXGBE_GPIE_VTMODE_MASK;

        switch (adapter->ring_feature[RING_F_VMDQ].mask) {
        case IXGBE_82599_VMDQ_8Q_MASK:
            gpie |= IXGBE_GPIE_VTMODE_16;
            break;
        case IXGBE_82599_VMDQ_4Q_MASK:
            gpie |= IXGBE_GPIE_VTMODE_32;
            break;
        default:
            gpie |= IXGBE_GPIE_VTMODE_64;
            break;
        }
    }

    /* Enable Thermal over heat sensor interrupt */
    if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82599EB:
            gpie |= IXGBE_SDP0_GPIEN;
            break;
        case ixgbe_mac_X540:
            gpie |= IXGBE_EIMS_TS;
            break;
        default:
            break;
        }
    }

    /* Enable fan failure interrupt */
    if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
        gpie |= IXGBE_SDP1_GPIEN;

    if (hw->mac.type == ixgbe_mac_82599EB) {
        gpie |= IXGBE_SDP1_GPIEN;
        gpie |= IXGBE_SDP2_GPIEN;
    }

    IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
}

static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int err;
    u32 ctrl_ext;

    ixgbe_get_hw_control(adapter);
    ixgbe_setup_gpie(adapter);

    if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
        ixgbe_configure_msix(adapter);
    else
        ixgbe_configure_msi_and_legacy(adapter);

    /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */
    if (hw->mac.ops.enable_tx_laser &&
        ((hw->phy.multispeed_fiber) ||
         ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
          (hw->mac.type == ixgbe_mac_82599EB))))
        hw->mac.ops.enable_tx_laser(hw);

    clear_bit(__IXGBE_DOWN, &adapter->state);
    ixgbe_napi_enable_all(adapter);

    if (ixgbe_is_sfp(hw)) {
        ixgbe_sfp_link_config(adapter);
    } else {
        err = ixgbe_non_sfp_link_config(hw);
        if (err)
            e_err(probe, "link_config FAILED %d\n", err);
    }

    /* clear any pending interrupts, may auto mask */
    IXGBE_READ_REG(hw, IXGBE_EICR);
    ixgbe_irq_enable(adapter, true, true);

    /*
     * If this adapter has a fan, check to see if we had a failure
     * before we enabled the interrupt.
     */
    if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
        u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
        if (esdp & IXGBE_ESDP_SDP1)
            e_crit(drv, "Fan has stopped, replace the adapter\n");
    }

    /* enable transmits */
    netif_tx_start_all_queues(adapter->netdev);

    /* bring the link up in the watchdog, this could race with our first
     * link up interrupt but shouldn't be a problem */
    adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
    adapter->link_check_timeout = jiffies;
    mod_timer(&adapter->service_timer, jiffies);

    /* Set PF Reset Done bit so PF/VF Mail Ops can work */
    ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
    ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
    IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
}

void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
{
    WARN_ON(in_interrupt());
    /* put off any impending NetWatchDogTimeout */
    adapter->netdev->trans_start = jiffies;

    while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
        usleep_range(1000, 2000);
    ixgbe_down(adapter);
    /*
     * If SR-IOV enabled then wait a bit before bringing the adapter
     * back up to give the VFs time to respond to the reset.  The
     * two second wait is based upon the watchdog timer cycle in
     * the VF driver.
     */
    if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
        msleep(2000);
    ixgbe_up(adapter);
    clear_bit(__IXGBE_RESETTING, &adapter->state);
}

void ixgbe_up(struct ixgbe_adapter *adapter)
{
    /* hardware has been reset, we need to reload some things */
    ixgbe_configure(adapter);

    ixgbe_up_complete(adapter);
}

void ixgbe_reset(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int err;

    /* lock SFP init bit to prevent race conditions with the watchdog */
    while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
        usleep_range(1000, 2000);

    /* clear all SFP and link config related flags while holding SFP_INIT */
    adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
                 IXGBE_FLAG2_SFP_NEEDS_RESET);
    adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;

    err = hw->mac.ops.init_hw(hw);
    switch (err) {
    case 0:
    case IXGBE_ERR_SFP_NOT_PRESENT:
    case IXGBE_ERR_SFP_NOT_SUPPORTED:
        break;
    case IXGBE_ERR_MASTER_REQUESTS_PENDING:
        e_dev_err("master disable timed out\n");
        break;
    case IXGBE_ERR_EEPROM_VERSION:
        /* We are running on a pre-production device, log a warning */
        e_dev_warn("This device is a pre-production adapter/LOM. "
               "Please be aware there may be issues associated with "
               "your hardware.  If you are experiencing problems "
               "please contact your Intel or hardware "
               "representative who provided you with this "
               "hardware.\n");
        break;
    default:
        e_dev_err("Hardware Error: %d\n", err);
    }

    clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);

    /* reprogram the RAR[0] in case user changed it. */
    hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV);

    /* update SAN MAC vmdq pool selection */
    if (hw->mac.san_mac_rar_index)
        hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
}

static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
{
    struct device *dev = rx_ring->dev;
    unsigned long size;
    u16 i;

    /* ring already cleared, nothing to do */
    if (!rx_ring->rx_buffer_info)
        return;

    /* Free all the Rx ring sk_buffs */
    for (i = 0; i < rx_ring->count; i++) {
        struct ixgbe_rx_buffer *rx_buffer;

        rx_buffer = &rx_ring->rx_buffer_info[i];
        if (rx_buffer->skb) {
            struct sk_buff *skb = rx_buffer->skb;
            if (IXGBE_CB(skb)->page_released) {
                dma_unmap_page(dev,
                           IXGBE_CB(skb)->dma,
                           ixgbe_rx_bufsz(rx_ring),
                           DMA_FROM_DEVICE);
                IXGBE_CB(skb)->page_released = false;
            }
            dev_kfree_skb(skb);
        }
        rx_buffer->skb = NULL;
        if (rx_buffer->dma)
            dma_unmap_page(dev, rx_buffer->dma,
                       ixgbe_rx_pg_size(rx_ring),
                       DMA_FROM_DEVICE);
        rx_buffer->dma = 0;
        if (rx_buffer->page)
            __free_pages(rx_buffer->page,
                     ixgbe_rx_pg_order(rx_ring));
        rx_buffer->page = NULL;
    }

    size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
    memset(rx_ring->rx_buffer_info, 0, size);

    /* Zero out the descriptor ring */
    memset(rx_ring->desc, 0, rx_ring->size);

    rx_ring->next_to_alloc = 0;
    rx_ring->next_to_clean = 0;
    rx_ring->next_to_use = 0;
}

static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
{
    struct ixgbe_tx_buffer *tx_buffer_info;
    unsigned long size;
    u16 i;

    /* ring already cleared, nothing to do */
    if (!tx_ring->tx_buffer_info)
        return;

    /* Free all the Tx ring sk_buffs */
    for (i = 0; i < tx_ring->count; i++) {
        tx_buffer_info = &tx_ring->tx_buffer_info[i];
        ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
    }

    netdev_tx_reset_queue(txring_txq(tx_ring));

    size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
    memset(tx_ring->tx_buffer_info, 0, size);

    /* Zero out the descriptor ring */
    memset(tx_ring->desc, 0, tx_ring->size);

    tx_ring->next_to_use = 0;
    tx_ring->next_to_clean = 0;
}

static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
{
    int i;

    for (i = 0; i < adapter->num_rx_queues; i++)
        ixgbe_clean_rx_ring(adapter->rx_ring[i]);
}

static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
{
    int i;

    for (i = 0; i < adapter->num_tx_queues; i++)
        ixgbe_clean_tx_ring(adapter->tx_ring[i]);
}

static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
{
    struct hlist_node *node, *node2;
    struct ixgbe_fdir_filter *filter;

    spin_lock(&adapter->fdir_perfect_lock);

    hlist_for_each_entry_safe(filter, node, node2,
                  &adapter->fdir_filter_list, fdir_node) {
        hlist_del(&filter->fdir_node);
        kfree(filter);
    }
    adapter->fdir_filter_count = 0;

    spin_unlock(&adapter->fdir_perfect_lock);
}

void ixgbe_down(struct ixgbe_adapter *adapter)
{
    struct net_device *netdev = adapter->netdev;
    struct ixgbe_hw *hw = &adapter->hw;
    u32 rxctrl;
    int i;

    /* signal that we are down to the interrupt handler */
    set_bit(__IXGBE_DOWN, &adapter->state);

    /* disable receives */
    rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
    IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);

    /* disable all enabled rx queues */
    for (i = 0; i < adapter->num_rx_queues; i++)
        /* this call also flushes the previous write */
        ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);

    usleep_range(10000, 20000);

    netif_tx_stop_all_queues(netdev);

    /* call carrier off first to avoid false dev_watchdog timeouts */
    netif_carrier_off(netdev);
    netif_tx_disable(netdev);

    ixgbe_irq_disable(adapter);

    ixgbe_napi_disable_all(adapter);

    adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
                 IXGBE_FLAG2_RESET_REQUESTED);
    adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;

    del_timer_sync(&adapter->service_timer);

    if (adapter->num_vfs) {
        /* Clear EITR Select mapping */
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);

        /* Mark all the VFs as inactive */
        for (i = 0 ; i < adapter->num_vfs; i++)
            adapter->vfinfo[i].clear_to_send = false;

        /* ping all the active vfs to let them know we are going down */
        ixgbe_ping_all_vfs(adapter);

        /* Disable all VFTE/VFRE TX/RX */
        ixgbe_disable_tx_rx(adapter);
    }

    /* disable transmits in the hardware now that interrupts are off */
    for (i = 0; i < adapter->num_tx_queues; i++) {
        u8 reg_idx = adapter->tx_ring[i]->reg_idx;
        IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
    }

    /* Disable the Tx DMA engine on 82599 and X540 */
    switch (hw->mac.type) {
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
                (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
                 ~IXGBE_DMATXCTL_TE));
        break;
    default:
        break;
    }

    if (!pci_channel_offline(adapter->pdev))
        ixgbe_reset(adapter);

    /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
    if (hw->mac.ops.disable_tx_laser &&
        ((hw->phy.multispeed_fiber) ||
         ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
          (hw->mac.type == ixgbe_mac_82599EB))))
        hw->mac.ops.disable_tx_laser(hw);

    ixgbe_clean_all_tx_rings(adapter);
    ixgbe_clean_all_rx_rings(adapter);

#ifdef CONFIG_IXGBE_DCA
    /* since we reset the hardware DCA settings were cleared */
    ixgbe_setup_dca(adapter);
#endif
}

static void ixgbe_tx_timeout(struct net_device *netdev)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);

    /* Do the reset outside of interrupt context */
    ixgbe_tx_timeout_reset(adapter);
}

static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    struct pci_dev *pdev = adapter->pdev;
    unsigned int rss;
#ifdef CONFIG_IXGBE_DCB
    int j;
    struct tc_configuration *tc;
#endif

    /* PCI config space info */

    hw->vendor_id = pdev->vendor;
    hw->device_id = pdev->device;
    hw->revision_id = pdev->revision;
    hw->subsystem_vendor_id = pdev->subsystem_vendor;
    hw->subsystem_device_id = pdev->subsystem_device;

    /* Set capability flags */
    rss = min_t(int, IXGBE_MAX_RSS_INDICES, num_online_cpus());
    adapter->ring_feature[RING_F_RSS].limit = rss;
    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        if (hw->device_id == IXGBE_DEV_ID_82598AT)
            adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
        adapter->max_q_vectors = MAX_Q_VECTORS_82598;
        break;
    case ixgbe_mac_X540:
        adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
    case ixgbe_mac_82599EB:
        adapter->max_q_vectors = MAX_Q_VECTORS_82599;
        adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
        adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
        if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
            adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
        /* Flow Director hash filters enabled */
        adapter->atr_sample_rate = 20;
        adapter->ring_feature[RING_F_FDIR].limit =
                             IXGBE_MAX_FDIR_INDICES;
        adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
#ifdef IXGBE_FCOE
        adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
        adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
#ifdef CONFIG_IXGBE_DCB
        /* Default traffic class to use for FCoE */
        adapter->fcoe.up = IXGBE_FCOE_DEFTC;
#endif
#endif /* IXGBE_FCOE */
        break;
    default:
        break;
    }

#ifdef IXGBE_FCOE
    /* FCoE support exists, always init the FCoE lock */
    spin_lock_init(&adapter->fcoe.lock);

#endif
    /* n-tuple support exists, always init our spinlock */
    spin_lock_init(&adapter->fdir_perfect_lock);

#ifdef CONFIG_IXGBE_DCB
    switch (hw->mac.type) {
    case ixgbe_mac_X540:
        adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
        adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
        break;
    default:
        adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
        adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
        break;
    }

    /* Configure DCB traffic classes */
    for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
        tc = &adapter->dcb_cfg.tc_config[j];
        tc->path[DCB_TX_CONFIG].bwg_id = 0;
        tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
        tc->path[DCB_RX_CONFIG].bwg_id = 0;
        tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
        tc->dcb_pfc = pfc_disabled;
    }

    /* Initialize default user to priority mapping, UPx->TC0 */
    tc = &adapter->dcb_cfg.tc_config[0];
    tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
    tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;

    adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
    adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
    adapter->dcb_cfg.pfc_mode_enable = false;
    adapter->dcb_set_bitmap = 0x00;
    adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
    memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
           sizeof(adapter->temp_dcb_cfg));

#endif

    /* default flow control settings */
    hw->fc.requested_mode = ixgbe_fc_full;
    hw->fc.current_mode = ixgbe_fc_full;    /* init for ethtool output */
    ixgbe_pbthresh_setup(adapter);
    hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
    hw->fc.send_xon = true;
    hw->fc.disable_fc_autoneg = false;

#ifdef CONFIG_PCI_IOV
    /* assign number of SR-IOV VFs */
    if (hw->mac.type != ixgbe_mac_82598EB)
        adapter->num_vfs = (max_vfs > 63) ? 0 : max_vfs;

#endif
    /* enable itr by default in dynamic mode */
    adapter->rx_itr_setting = 1;
    adapter->tx_itr_setting = 1;

    /* set default ring sizes */
    adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
    adapter->rx_ring_count = IXGBE_DEFAULT_RXD;

    /* set default work limits */
    adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;

    /* initialize eeprom parameters */
    if (ixgbe_init_eeprom_params_generic(hw)) {
        e_dev_err("EEPROM initialization failed\n");
        return -EIO;
    }

    set_bit(__IXGBE_DOWN, &adapter->state);

    return 0;
}

int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
{
    struct device *dev = tx_ring->dev;
    int orig_node = dev_to_node(dev);
    int numa_node = -1;
    int size;

    size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;

    if (tx_ring->q_vector)
        numa_node = tx_ring->q_vector->numa_node;

    tx_ring->tx_buffer_info = vzalloc_node(size, numa_node);
    if (!tx_ring->tx_buffer_info)
        tx_ring->tx_buffer_info = vzalloc(size);
    if (!tx_ring->tx_buffer_info)
        goto err;

    /* round up to nearest 4K */
    tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
    tx_ring->size = ALIGN(tx_ring->size, 4096);

    set_dev_node(dev, numa_node);
    tx_ring->desc = dma_alloc_coherent(dev,
                       tx_ring->size,
                       &tx_ring->dma,
                       GFP_KERNEL);
    set_dev_node(dev, orig_node);
    if (!tx_ring->desc)
        tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
                           &tx_ring->dma, GFP_KERNEL);
    if (!tx_ring->desc)
        goto err;

    tx_ring->next_to_use = 0;
    tx_ring->next_to_clean = 0;
    return 0;

err:
    vfree(tx_ring->tx_buffer_info);
    tx_ring->tx_buffer_info = NULL;
    dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
    return -ENOMEM;
}

static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
{
    int i, err = 0;

    for (i = 0; i < adapter->num_tx_queues; i++) {
        err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
        if (!err)
            continue;

        e_err(probe, "Allocation for Tx Queue %u failed\n", i);
        goto err_setup_tx;
    }

    return 0;
err_setup_tx:
    /* rewind the index freeing the rings as we go */
    while (i--)
        ixgbe_free_tx_resources(adapter->tx_ring[i]);
    return err;
}

int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
{
    struct device *dev = rx_ring->dev;
    int orig_node = dev_to_node(dev);
    int numa_node = -1;
    int size;

    size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;

    if (rx_ring->q_vector)
        numa_node = rx_ring->q_vector->numa_node;

    rx_ring->rx_buffer_info = vzalloc_node(size, numa_node);
    if (!rx_ring->rx_buffer_info)
        rx_ring->rx_buffer_info = vzalloc(size);
    if (!rx_ring->rx_buffer_info)
        goto err;

    /* Round up to nearest 4K */
    rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
    rx_ring->size = ALIGN(rx_ring->size, 4096);

    set_dev_node(dev, numa_node);
    rx_ring->desc = dma_alloc_coherent(dev,
                       rx_ring->size,
                       &rx_ring->dma,
                       GFP_KERNEL);
    set_dev_node(dev, orig_node);
    if (!rx_ring->desc)
        rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
                           &rx_ring->dma, GFP_KERNEL);
    if (!rx_ring->desc)
        goto err;

    rx_ring->next_to_clean = 0;
    rx_ring->next_to_use = 0;

    return 0;
err:
    vfree(rx_ring->rx_buffer_info);
    rx_ring->rx_buffer_info = NULL;
    dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
    return -ENOMEM;
}

static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
{
    int i, err = 0;

    for (i = 0; i < adapter->num_rx_queues; i++) {
        err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
        if (!err)
            continue;

        e_err(probe, "Allocation for Rx Queue %u failed\n", i);
        goto err_setup_rx;
    }

#ifdef IXGBE_FCOE
    err = ixgbe_setup_fcoe_ddp_resources(adapter);
    if (!err)
#endif
        return 0;
err_setup_rx:
    /* rewind the index freeing the rings as we go */
    while (i--)
        ixgbe_free_rx_resources(adapter->rx_ring[i]);
    return err;
}

void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
{
    ixgbe_clean_tx_ring(tx_ring);

    vfree(tx_ring->tx_buffer_info);
    tx_ring->tx_buffer_info = NULL;

    /* if not set, then don't free */
    if (!tx_ring->desc)
        return;

    dma_free_coherent(tx_ring->dev, tx_ring->size,
              tx_ring->desc, tx_ring->dma);

    tx_ring->desc = NULL;
}

static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
{
    int i;

    for (i = 0; i < adapter->num_tx_queues; i++)
        if (adapter->tx_ring[i]->desc)
            ixgbe_free_tx_resources(adapter->tx_ring[i]);
}

void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
{
    ixgbe_clean_rx_ring(rx_ring);

    vfree(rx_ring->rx_buffer_info);
    rx_ring->rx_buffer_info = NULL;

    /* if not set, then don't free */
    if (!rx_ring->desc)
        return;

    dma_free_coherent(rx_ring->dev, rx_ring->size,
              rx_ring->desc, rx_ring->dma);

    rx_ring->desc = NULL;
}

static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
{
    int i;

#ifdef IXGBE_FCOE
    ixgbe_free_fcoe_ddp_resources(adapter);

#endif
    for (i = 0; i < adapter->num_rx_queues; i++)
        if (adapter->rx_ring[i]->desc)
            ixgbe_free_rx_resources(adapter->rx_ring[i]);
}

static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);
    int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;

    /* MTU < 68 is an error and causes problems on some kernels */
    if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
        return -EINVAL;

    /*
     * For 82599EB we cannot allow PF to change MTU greater than 1500
     * in SR-IOV mode as it may cause buffer overruns in guest VFs that
     * don't allocate and chain buffers correctly.
     */
    if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
        (adapter->hw.mac.type == ixgbe_mac_82599EB) &&
        (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
            return -EINVAL;

    e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);

    /* must set new MTU before calling down or up */
    netdev->mtu = new_mtu;

    if (netif_running(netdev))
        ixgbe_reinit_locked(adapter);

    return 0;
}

static int ixgbe_open(struct net_device *netdev)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);
    int err;

    /* disallow open during test */
    if (test_bit(__IXGBE_TESTING, &adapter->state))
        return -EBUSY;

    netif_carrier_off(netdev);

    /* allocate transmit descriptors */
    err = ixgbe_setup_all_tx_resources(adapter);
    if (err)
        goto err_setup_tx;

    /* allocate receive descriptors */
    err = ixgbe_setup_all_rx_resources(adapter);
    if (err)
        goto err_setup_rx;

    ixgbe_configure(adapter);

    err = ixgbe_request_irq(adapter);
    if (err)
        goto err_req_irq;

    /* Notify the stack of the actual queue counts. */
    err = netif_set_real_num_tx_queues(netdev,
                       adapter->num_rx_pools > 1 ? 1 :
                       adapter->num_tx_queues);
    if (err)
        goto err_set_queues;


    err = netif_set_real_num_rx_queues(netdev,
                       adapter->num_rx_pools > 1 ? 1 :
                       adapter->num_rx_queues);
    if (err)
        goto err_set_queues;

    ixgbe_up_complete(adapter);

    return 0;

err_set_queues:
    ixgbe_free_irq(adapter);
err_req_irq:
    ixgbe_free_all_rx_resources(adapter);
err_setup_rx:
    ixgbe_free_all_tx_resources(adapter);
err_setup_tx:
    ixgbe_reset(adapter);

    return err;
}

static int ixgbe_close(struct net_device *netdev)
{
    struct ixgbe_adapter *adapter = netdev_priv(netdev);

    ixgbe_down(adapter);
    ixgbe_free_irq(adapter);

    ixgbe_fdir_filter_exit(adapter);

    ixgbe_free_all_tx_resources(adapter);
    ixgbe_free_all_rx_resources(adapter);

    ixgbe_release_hw_control(adapter);

    return 0;
}

#ifdef CONFIG_PM
static int ixgbe_resume(struct pci_dev *pdev)
{
    struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
    struct net_device *netdev = adapter->netdev;
    u32 err;

    pci_set_power_state(pdev, PCI_D0);
    pci_restore_state(pdev);
    /*
     * pci_restore_state clears dev->state_saved so call
     * pci_save_state to restore it.
     */
    pci_save_state(pdev);

    err = pci_enable_device_mem(pdev);
    if (err) {
        e_dev_err("Cannot enable PCI device from suspend\n");
        return err;
    }
    pci_set_master(pdev);

    pci_wake_from_d3(pdev, false);

    ixgbe_reset(adapter);

    IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);

    rtnl_lock();
    err = ixgbe_init_interrupt_scheme(adapter);
    if (!err && netif_running(netdev))
        err = ixgbe_open(netdev);

    rtnl_unlock();

    if (err)
        return err;

    netif_device_attach(netdev);

    return 0;
}
#endif /* CONFIG_PM */

static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
{
    struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
    struct net_device *netdev = adapter->netdev;
    struct ixgbe_hw *hw = &adapter->hw;
    u32 ctrl, fctrl;
    u32 wufc = adapter->wol;
#ifdef CONFIG_PM
    int retval = 0;
#endif

    netif_device_detach(netdev);

    if (netif_running(netdev)) {
        rtnl_lock();
        ixgbe_down(adapter);
        ixgbe_free_irq(adapter);
        ixgbe_free_all_tx_resources(adapter);
        ixgbe_free_all_rx_resources(adapter);
        rtnl_unlock();
    }

    ixgbe_clear_interrupt_scheme(adapter);

#ifdef CONFIG_PM
    retval = pci_save_state(pdev);
    if (retval)
        return retval;

#endif
    if (wufc) {
        ixgbe_set_rx_mode(netdev);

        /*
         * enable the optics for both mult-speed fiber and
         * 82599 SFP+ fiber as we can WoL.
         */
        if (hw->mac.ops.enable_tx_laser &&
            (hw->phy.multispeed_fiber ||
            (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber &&
             hw->mac.type == ixgbe_mac_82599EB)))
            hw->mac.ops.enable_tx_laser(hw);

        /* turn on all-multi mode if wake on multicast is enabled */
        if (wufc & IXGBE_WUFC_MC) {
            fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
            fctrl |= IXGBE_FCTRL_MPE;
            IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
        }

        ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
        ctrl |= IXGBE_CTRL_GIO_DIS;
        IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);

        IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
    } else {
        IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
        IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
    }

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        pci_wake_from_d3(pdev, false);
        break;
    case ixgbe_mac_82599EB:
    case ixgbe_mac_X540:
        pci_wake_from_d3(pdev, !!wufc);
        break;
    default:
        break;
    }

    *enable_wake = !!wufc;

    ixgbe_release_hw_control(adapter);

    pci_disable_device(pdev);

    return 0;
}

#ifdef CONFIG_PM
static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
{
    int retval;
    bool wake;

    retval = __ixgbe_shutdown(pdev, &wake);
    if (retval)
        return retval;

    if (wake) {
        pci_prepare_to_sleep(pdev);
    } else {
        pci_wake_from_d3(pdev, false);
        pci_set_power_state(pdev, PCI_D3hot);
    }

    return 0;
}
#endif /* CONFIG_PM */

static void ixgbe_shutdown(struct pci_dev *pdev)
{
    bool wake;

    __ixgbe_shutdown(pdev, &wake);

    if (system_state == SYSTEM_POWER_OFF) {
        pci_wake_from_d3(pdev, wake);
        pci_set_power_state(pdev, PCI_D3hot);
    }
}

void ixgbe_update_stats(struct ixgbe_adapter *adapter)
{
    struct net_device *netdev = adapter->netdev;
    struct ixgbe_hw *hw = &adapter->hw;
    struct ixgbe_hw_stats *hwstats = &adapter->stats;
    u64 total_mpc = 0;
    u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
    u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
    u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
    u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;

    if (test_bit(__IXGBE_DOWN, &adapter->state) ||
        test_bit(__IXGBE_RESETTING, &adapter->state))
        return;

    if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
        u64 rsc_count = 0;
        u64 rsc_flush = 0;
        for (i = 0; i < adapter->num_rx_queues; i++) {
            rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
            rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
        }
        adapter->rsc_total_count = rsc_count;
        adapter->rsc_total_flush = rsc_flush;
    }

    for (i = 0; i < adapter->num_rx_queues; i++) {
        struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
        non_eop_descs += rx_ring->rx_stats.non_eop_descs;
        alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
        alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
        hw_csum_rx_error += rx_ring->rx_stats.csum_err;
        bytes += rx_ring->stats.bytes;
        packets += rx_ring->stats.packets;
    }
    adapter->non_eop_descs = non_eop_descs;
    adapter->alloc_rx_page_failed = alloc_rx_page_failed;
    adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
    adapter->hw_csum_rx_error = hw_csum_rx_error;
    netdev->stats.rx_bytes = bytes;
    netdev->stats.rx_packets = packets;

    bytes = 0;
    packets = 0;
    /* gather some stats to the adapter struct that are per queue */
    for (i = 0; i < adapter->num_tx_queues; i++) {
        struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
        restart_queue += tx_ring->tx_stats.restart_queue;
        tx_busy += tx_ring->tx_stats.tx_busy;
        bytes += tx_ring->stats.bytes;
        packets += tx_ring->stats.packets;
    }
    adapter->restart_queue = restart_queue;
    adapter->tx_busy = tx_busy;
    netdev->stats.tx_bytes = bytes;
    netdev->stats.tx_packets = packets;

    hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);

    /* 8 register reads */
    for (i = 0; i < 8; i++) {
        /* for packet buffers not used, the register should read 0 */
        mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
        missed_rx += mpc;
        hwstats->mpc[i] += mpc;
        total_mpc += hwstats->mpc[i];
        hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
        hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
            hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
            hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
            hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
            hwstats->pxonrxc[i] +=
                IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
            break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
            hwstats->pxonrxc[i] +=
                IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
            break;
        default:
            break;
        }
    }

    /*16 register reads */
    for (i = 0; i < 16; i++) {
        hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
        hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
        if ((hw->mac.type == ixgbe_mac_82599EB) ||
            (hw->mac.type == ixgbe_mac_X540)) {
            hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
            IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
            hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
            IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
        }
    }

    hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
    /* work around hardware counting issue */
    hwstats->gprc -= missed_rx;

    ixgbe_update_xoff_received(adapter);

    /* 82598 hardware only has a 32 bit counter in the high register */
    switch (hw->mac.type) {
    case ixgbe_mac_82598EB:
        hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
        hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
        hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
        hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
        break;
    case ixgbe_mac_X540:
        /* OS2BMC stats are X540 only*/
        hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
        hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
        hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
        hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
    case ixgbe_mac_82599EB:
        for (i = 0; i < 16; i++)
            adapter->hw_rx_no_dma_resources +=
                         IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
        hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
        IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
        hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
        IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
        hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
        IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
        hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
        hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
        hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
#ifdef IXGBE_FCOE
        hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
        hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
        hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
        hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
        hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
        hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
        /* Add up per cpu counters for total ddp aloc fail */
        if (adapter->fcoe.ddp_pool) {
            struct ixgbe_fcoe *fcoe = &adapter->fcoe;
            struct ixgbe_fcoe_ddp_pool *ddp_pool;
            unsigned int cpu;
            u64 noddp = 0, noddp_ext_buff = 0;
            for_each_possible_cpu(cpu) {
                ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);
                noddp += ddp_pool->noddp;
                noddp_ext_buff += ddp_pool->noddp_ext_buff;
            }
            hwstats->fcoe_noddp = noddp;
            hwstats->fcoe_noddp_ext_buff = noddp_ext_buff;
        }
#endif /* IXGBE_FCOE */
        break;
    default:
        break;
    }
    bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
    hwstats->bprc += bprc;
    hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
    if (hw->mac.type == ixgbe_mac_82598EB)
        hwstats->mprc -= bprc;
    hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
    hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
    hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
    hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
    hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
    hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
    hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
    hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
    lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
    hwstats->lxontxc += lxon;
    lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
    hwstats->lxofftxc += lxoff;
    hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
    hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
    /*
     * 82598 errata - tx of flow control packets is included in tx counters
     */
    xon_off_tot = lxon + lxoff;
    hwstats->gptc -= xon_off_tot;
    hwstats->mptc -= xon_off_tot;
    hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
    hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
    hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
    hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
    hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
    hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
    hwstats->ptc64 -= xon_off_tot;
    hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
    hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
    hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
    hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
    hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
    hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);

    /* Fill out the OS statistics structure */
    netdev->stats.multicast = hwstats->mprc;

    /* Rx Errors */
    netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
    netdev->stats.rx_dropped = 0;
    netdev->stats.rx_length_errors = hwstats->rlec;
    netdev->stats.rx_crc_errors = hwstats->crcerrs;
    netdev->stats.rx_missed_errors = total_mpc;
}

static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    int i;

    if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
        return;

    adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;

    /* if interface is down do nothing */
    if (test_bit(__IXGBE_DOWN, &adapter->state))
        return;

    /* do nothing if we are not using signature filters */
    if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
        return;

    adapter->fdir_overflow++;

    if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
        for (i = 0; i < adapter->num_tx_queues; i++)
            set_bit(__IXGBE_TX_FDIR_INIT_DONE,
                    &(adapter->tx_ring[i]->state));
        /* re-enable flow director interrupts */
        IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
    } else {
        e_err(probe, "failed to finish FDIR re-initialization, "
              "ignored adding FDIR ATR filters\n");
    }
}

static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u64 eics = 0;
    int i;

    /* If we're down or resetting, just bail */
    if (test_bit(__IXGBE_DOWN, &adapter->state) ||
        test_bit(__IXGBE_RESETTING, &adapter->state))
        return;

    /* Force detection of hung controller */
    if (netif_carrier_ok(adapter->netdev)) {
        for (i = 0; i < adapter->num_tx_queues; i++)
            set_check_for_tx_hang(adapter->tx_ring[i]);
    }

    if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
        /*
         * for legacy and MSI interrupts don't set any bits
         * that are enabled for EIAM, because this operation
         * would set *both* EIMS and EICS for any bit in EIAM
         */
        IXGBE_WRITE_REG(hw, IXGBE_EICS,
            (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
    } else {
        /* get one bit for every active tx/rx interrupt vector */
        for (i = 0; i < adapter->num_q_vectors; i++) {
            struct ixgbe_q_vector *qv = adapter->q_vector[i];
            if (qv->rx.ring || qv->tx.ring)
                eics |= ((u64)1 << i);
        }
    }

    /* Cause software interrupt to ensure rings are cleaned */
    ixgbe_irq_rearm_queues(adapter, eics);

}

static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 link_speed = adapter->link_speed;
    bool link_up = adapter->link_up;
    bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

    if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
        return;

    if (hw->mac.ops.check_link) {
        hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
    } else {
        /* always assume link is up, if no check link function */
        link_speed = IXGBE_LINK_SPEED_10GB_FULL;
        link_up = true;
    }

    if (adapter->ixgbe_ieee_pfc)
        pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

    if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {
        hw->mac.ops.fc_enable(hw);
        ixgbe_set_rx_drop_en(adapter);
    }

    if (link_up ||
        time_after(jiffies, (adapter->link_check_timeout +
                 IXGBE_TRY_LINK_TIMEOUT))) {
        adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
        IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
        IXGBE_WRITE_FLUSH(hw);
    }

    adapter->link_up = link_up;
    adapter->link_speed = link_speed;
}

static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
{
    struct net_device *netdev = adapter->netdev;
    struct ixgbe_hw *hw = &adapter->hw;
    u32 link_speed = adapter->link_speed;
    bool flow_rx, flow_tx;

    /* only continue if link was previously down */
    if (netif_carrier_ok(netdev))
        return;

    adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;

    switch (hw->mac.type) {
    case ixgbe_mac_82598EB: {
        u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
        u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
        flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
        flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
    }
        break;
    case ixgbe_mac_X540:
    case ixgbe_mac_82599EB: {
        u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
        u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
        flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
        flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
    }
        break;
    default:
        flow_tx = false;
        flow_rx = false;
        break;
    }

#ifdef CONFIG_IXGBE_PTP
    ixgbe_ptp_start_cyclecounter(adapter);
#endif

    e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
           (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
           "10 Gbps" :
           (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
           "1 Gbps" :
           (link_speed == IXGBE_LINK_SPEED_100_FULL ?
           "100 Mbps" :
           "unknown speed"))),
           ((flow_rx && flow_tx) ? "RX/TX" :
           (flow_rx ? "RX" :
           (flow_tx ? "TX" : "None"))));

    netif_carrier_on(netdev);
    ixgbe_check_vf_rate_limit(adapter);

    /* ping all the active vfs to let them know link has changed */
    ixgbe_ping_all_vfs(adapter);
}

static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)
{
    struct net_device *netdev = adapter->netdev;
    struct ixgbe_hw *hw = &adapter->hw;

    adapter->link_up = false;
    adapter->link_speed = 0;

    /* only continue if link was up previously */
    if (!netif_carrier_ok(netdev))
        return;

    /* poll for SFP+ cable when link is down */
    if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
        adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;

#ifdef CONFIG_IXGBE_PTP
    ixgbe_ptp_start_cyclecounter(adapter);
#endif

    e_info(drv, "NIC Link is Down\n");
    netif_carrier_off(netdev);

    /* ping all the active vfs to let them know link has changed */
    ixgbe_ping_all_vfs(adapter);
}

static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
{
    int i;
    int some_tx_pending = 0;

    if (!netif_carrier_ok(adapter->netdev)) {
        for (i = 0; i < adapter->num_tx_queues; i++) {
            struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
            if (tx_ring->next_to_use != tx_ring->next_to_clean) {
                some_tx_pending = 1;
                break;
            }
        }

        if (some_tx_pending) {
            /* We've lost link, so the controller stops DMA,
             * but we've got queued Tx work that's never going
             * to get done, so reset controller to flush Tx.
             * (Do the reset outside of interrupt context).
             */
            adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
        }
    }
}

static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
{
    u32 ssvpc;

    /* Do not perform spoof check for 82598 or if not in IOV mode */
    if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
        adapter->num_vfs == 0)
        return;

    ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);

    /*
     * ssvpc register is cleared on read, if zero then no
     * spoofed packets in the last interval.
     */
    if (!ssvpc)
        return;

    e_warn(drv, "%u Spoofed packets detected\n", ssvpc);
}

static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
{
    /* if interface is down do nothing */
    if (test_bit(__IXGBE_DOWN, &adapter->state) ||
        test_bit(__IXGBE_RESETTING, &adapter->state))
        return;

    ixgbe_watchdog_update_link(adapter);

    if (adapter->link_up)
        ixgbe_watchdog_link_is_up(adapter);
    else
        ixgbe_watchdog_link_is_down(adapter);

    ixgbe_spoof_check(adapter);
    ixgbe_update_stats(adapter);

    ixgbe_watchdog_flush_tx(adapter);
}

static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    s32 err;

    /* not searching for SFP so there is nothing to do here */
    if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
        !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
        return;

    /* someone else is in init, wait until next service event */
    if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
        return;

    err = hw->phy.ops.identify_sfp(hw);
    if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
        goto sfp_out;

    if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
        /* If no cable is present, then we need to reset
         * the next time we find a good cable. */
        adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
    }

    /* exit on error */
    if (err)
        goto sfp_out;

    /* exit if reset not needed */
    if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
        goto sfp_out;

    adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;

    /*
     * A module may be identified correctly, but the EEPROM may not have
     * support for that module.  setup_sfp() will fail in that case, so
     * we should not allow that module to load.
     */
    if (hw->mac.type == ixgbe_mac_82598EB)
        err = hw->phy.ops.reset(hw);
    else
        err = hw->mac.ops.setup_sfp(hw);

    if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
        goto sfp_out;

    adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
    e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);

sfp_out:
    clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);

    if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
        (adapter->netdev->reg_state == NETREG_REGISTERED)) {
        e_dev_err("failed to initialize because an unsupported "
              "SFP+ module type was detected.\n");
        e_dev_err("Reload the driver after installing a "
              "supported module.\n");
        unregister_netdev(adapter->netdev);
    }
}

static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
{
    struct ixgbe_hw *hw = &adapter->hw;
    u32 autoneg;
    bool negotiation;

    if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
        return;

    /* someone else is in init, wait until next service event */
    if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
        return;

    adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;

    autoneg = hw->phy.autoneg_advertised;
    if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
        hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
    if (hw->mac.ops.setup_link)
        hw->mac.ops.setup_link(hw, autoneg, negotiation, true);

    adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
    adapter->link_check_timeout = jiffies;
    clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
}

#ifdef CONFIG_PCI_IOV
static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
{
    int vf;
    struct ixgbe_hw *hw = &adapter->hw;
    struct net_device *netdev = adapter->netdev;
    u32 gpc;
    u32 ciaa, ciad;

    gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
    if (gpc) /* If incrementing then no need for the check below */
        return;
    /*
     * Check to see if a bad DMA write target from an errant or
     * malicious VF has caused a PCIe error.  If so then we can
     * issue a VFLR to the offending VF(s) and then resume without
     * requesting a full slot reset.
     */

    for (vf = 0; vf < adapter->num_vfs; vf++) {
        ciaa = (vf << 16) | 0x80000000;
        /* 32 bit read so align, we really want status at offset 6 */
        ciaa |= PCI_COMMAND;
        IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
        ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599);
        ciaa &= 0x7FFFFFFF;
        /* disable debug mode asap after reading data */
        IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
        /* Get the upper 16 bits which will be the PCI status reg */
        ciad >>= 16;
        if (ciad & PCI_STATUS_REC_MASTER_ABORT) {
            netdev_err(netdev, "VF %d Hung DMA\n", vf);
            /* Issue VFLR */
            ciaa = (vf << 16) | 0x80000000;
            ciaa |= 0xA8;
            IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
            ciad = 0x00008000;  /* VFLR */
            IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad);
            ciaa &= 0x7FFFFFFF;
            IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
        }
    }
}

#endif

static void ixgbe_service_timer(unsigned long data)
{
    struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
    unsigned long next_event_offset;
    bool ready = true;

    /* poll faster when waiting for link */
    if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
        next_event_offset = HZ / 10;
    else
        next_event_offset = HZ * 2;

#ifdef CONFIG_PCI_IOV
    /*
     * don't bother with SR-IOV VF DMA hang check if there are
     * no VFs or the link is down
     */
    if (!adapter->num_vfs ||
        (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
        goto normal_timer_service;

    /* If we have VFs allocated then we must check for DMA hangs */
    ixgbe_check_for_bad_vf(adapter);
    next_event_offset = HZ / 50;
    adapter->timer_event_accumulator++;

    if (adapter->timer_event_accumulator >= 100)
        adapter->timer_event_accumulator = 0;
    else
        ready = false;

normal_timer_service:
#endif
    /* Reset the timer */
    mod_timer(&adapter->service_timer, next_event_offset + jiffies);

    if (ready)
        ixgbe_service_event_schedule(adapter);
}

static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
{
    if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
        return;

    adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;

    /* If we're already down or resetting, just bail */
    if (test_bit(__IXGBE_DOWN, &adapter->state) ||
        test_bit(__IXGBE_RESETTING, &adapter->state))
        return;

    ixgbe_dump(adapter);
    netdev_err(adapter->netdev, "Reset adapter\n");
    adapter->tx_timeout_count++;

    ixgbe_reinit_locked(adapter);
}

static void ixgbe_service_task(struct work_struct *work)
{
    struct ixgbe_adapter *adapter = container_of(work,
                             struct ixgbe_adapter,
                             service_task);

    ixgbe_reset_subtask(adapter);
    ixgbe_sfp_detection_subtask(adapter);
    ixgbe_sfp_link_config_subtask(adapter);
    ixgbe_check_overtemp_subtask(adapter);
    ixgbe_watchdog_subtask(adapter);
    ixgbe_fdir_reinit_subtask(adapter);
    ixgbe_check_hang_subtask(adapter);
#ifdef CONFIG_IXGBE_PTP
    ixgbe_ptp_overflow_check(adapter);
#endif

    ixgbe_service_event_complete(adapter);
}

static int ixgbe_tso(struct ixgbe_ring *tx_ring,
             struct ixgbe_tx_buffer *first,
             u8 *hdr_len)
{
    struct sk_buff *skb = first->skb;
    u32 vlan_macip_lens, type_tucmd;
    u32 mss_l4len_idx, l4len;

    if (!skb_is_gso(skb))
        return 0;

    if (skb_header_cloned(skb)) {
        int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
        if (err)
            return err;
    }

    /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
    type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;

    if (first->