cxgb3i.c

Go to the documentation of this file.

/*
 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
 *
 * Copyright (C) 2003-2008 Chelsio Communications.  All rights reserved.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
 * release for licensing terms and conditions.
 *
 * Written by:  Dimitris Michailidis ([email protected])
 *      Karen Xie ([email protected])
 */

#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <scsi/scsi_host.h>

#include "common.h"
#include "t3_cpl.h"
#include "t3cdev.h"
#include "cxgb3_defs.h"
#include "cxgb3_ctl_defs.h"
#include "cxgb3_offload.h"
#include "firmware_exports.h"
#include "cxgb3i.h"

static unsigned int dbg_level;
#include "../libcxgbi.h"

#define DRV_MODULE_NAME         "cxgb3i"
#define DRV_MODULE_DESC         "Chelsio T3 iSCSI Driver"
#define DRV_MODULE_VERSION  "2.0.0"
#define DRV_MODULE_RELDATE  "Jun. 2010"

static char version[] =
    DRV_MODULE_DESC " " DRV_MODULE_NAME
    " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

MODULE_AUTHOR("Chelsio Communications, Inc.");
MODULE_DESCRIPTION(DRV_MODULE_DESC);
MODULE_VERSION(DRV_MODULE_VERSION);
MODULE_LICENSE("GPL");

module_param(dbg_level, uint, 0644);
MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");

static int cxgb3i_rcv_win = 256 * 1024;
module_param(cxgb3i_rcv_win, int, 0644);
MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");

static int cxgb3i_snd_win = 128 * 1024;
module_param(cxgb3i_snd_win, int, 0644);
MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");

static int cxgb3i_rx_credit_thres = 10 * 1024;
module_param(cxgb3i_rx_credit_thres, int, 0644);
MODULE_PARM_DESC(rx_credit_thres,
         "RX credits return threshold in bytes (default=10KB)");

static unsigned int cxgb3i_max_connect = 8 * 1024;
module_param(cxgb3i_max_connect, uint, 0644);
MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");

static unsigned int cxgb3i_sport_base = 20000;
module_param(cxgb3i_sport_base, uint, 0644);
MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");

static void cxgb3i_dev_open(struct t3cdev *);
static void cxgb3i_dev_close(struct t3cdev *);
static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);

static struct cxgb3_client t3_client = {
    .name = DRV_MODULE_NAME,
    .handlers = cxgb3i_cpl_handlers,
    .add = cxgb3i_dev_open,
    .remove = cxgb3i_dev_close,
    .event_handler = cxgb3i_dev_event_handler,
};

static struct scsi_host_template cxgb3i_host_template = {
    .module     = THIS_MODULE,
    .name       = DRV_MODULE_NAME,
    .proc_name  = DRV_MODULE_NAME,
    .can_queue  = CXGB3I_SCSI_HOST_QDEPTH,
    .queuecommand   = iscsi_queuecommand,
    .change_queue_depth = iscsi_change_queue_depth,
    .sg_tablesize   = SG_ALL,
    .max_sectors    = 0xFFFF,
    .cmd_per_lun    = ISCSI_DEF_CMD_PER_LUN,
    .eh_abort_handler = iscsi_eh_abort,
    .eh_device_reset_handler = iscsi_eh_device_reset,
    .eh_target_reset_handler = iscsi_eh_recover_target,
    .target_alloc   = iscsi_target_alloc,
    .use_clustering = DISABLE_CLUSTERING,
    .this_id    = -1,
};

static struct iscsi_transport cxgb3i_iscsi_transport = {
    .owner      = THIS_MODULE,
    .name       = DRV_MODULE_NAME,
    /* owner and name should be set already */
    .caps       = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
                | CAP_DATADGST | CAP_DIGEST_OFFLOAD |
                CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
    .attr_is_visible    = cxgbi_attr_is_visible,
    .get_host_param = cxgbi_get_host_param,
    .set_host_param = cxgbi_set_host_param,
    /* session management */
    .create_session = cxgbi_create_session,
    .destroy_session    = cxgbi_destroy_session,
    .get_session_param = iscsi_session_get_param,
    /* connection management */
    .create_conn    = cxgbi_create_conn,
    .bind_conn  = cxgbi_bind_conn,
    .destroy_conn   = iscsi_tcp_conn_teardown,
    .start_conn = iscsi_conn_start,
    .stop_conn  = iscsi_conn_stop,
    .get_conn_param = iscsi_conn_get_param,
    .set_param  = cxgbi_set_conn_param,
    .get_stats  = cxgbi_get_conn_stats,
    /* pdu xmit req from user space */
    .send_pdu   = iscsi_conn_send_pdu,
    /* task */
    .init_task  = iscsi_tcp_task_init,
    .xmit_task  = iscsi_tcp_task_xmit,
    .cleanup_task   = cxgbi_cleanup_task,
    /* pdu */
    .alloc_pdu  = cxgbi_conn_alloc_pdu,
    .init_pdu   = cxgbi_conn_init_pdu,
    .xmit_pdu   = cxgbi_conn_xmit_pdu,
    .parse_pdu_itt  = cxgbi_parse_pdu_itt,
    /* TCP connect/disconnect */
    .get_ep_param   = cxgbi_get_ep_param,
    .ep_connect = cxgbi_ep_connect,
    .ep_poll    = cxgbi_ep_poll,
    .ep_disconnect  = cxgbi_ep_disconnect,
    /* Error recovery timeout call */
    .session_recovery_timedout = iscsi_session_recovery_timedout,
};

static struct scsi_transport_template *cxgb3i_stt;

/*
 * CPL (Chelsio Protocol Language) defines a message passing interface between
 * the host driver and Chelsio asic.
 * The section below implments CPLs that related to iscsi tcp connection
 * open/close/abort and data send/receive.
 */

static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);

static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
                  const struct l2t_entry *e)
{
    unsigned int wscale = cxgbi_sock_compute_wscale(cxgb3i_rcv_win);
    struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;

    skb->priority = CPL_PRIORITY_SETUP;

    req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
    OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
    req->local_port = csk->saddr.sin_port;
    req->peer_port = csk->daddr.sin_port;
    req->local_ip = csk->saddr.sin_addr.s_addr;
    req->peer_ip = csk->daddr.sin_addr.s_addr;

    req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
            V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
            V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
    req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
            V_RCV_BUFSIZ(cxgb3i_rcv_win>>10));

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
        csk, csk->state, csk->flags, csk->atid,
        &req->local_ip, ntohs(req->local_port),
        &req->peer_ip, ntohs(req->peer_port),
        csk->mss_idx, e->idx, e->smt_idx);

    l2t_send(csk->cdev->lldev, skb, csk->l2t);
}

static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
{
    cxgbi_sock_act_open_req_arp_failure(NULL, skb);
}

/*
 * CPL connection close request: host ->
 *
 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
 * the write queue (i.e., after any unsent txt data).
 */
static void send_close_req(struct cxgbi_sock *csk)
{
    struct sk_buff *skb = csk->cpl_close;
    struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
    unsigned int tid = csk->tid;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u.\n",
        csk, csk->state, csk->flags, csk->tid);

    csk->cpl_close = NULL;
    req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
    req->wr.wr_lo = htonl(V_WR_TID(tid));
    OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
    req->rsvd = htonl(csk->write_seq);

    cxgbi_sock_skb_entail(csk, skb);
    if (csk->state >= CTP_ESTABLISHED)
        push_tx_frames(csk, 1);
}

/*
 * CPL connection abort request: host ->
 *
 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
 * for the same connection and also that we do not try to send a message
 * after the connection has closed.
 */
static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
{
    struct cpl_abort_req *req = cplhdr(skb);

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "t3dev 0x%p, tid %u, skb 0x%p.\n",
        tdev, GET_TID(req), skb);
    req->cmd = CPL_ABORT_NO_RST;
    cxgb3_ofld_send(tdev, skb);
}

static void send_abort_req(struct cxgbi_sock *csk)
{
    struct sk_buff *skb = csk->cpl_abort_req;
    struct cpl_abort_req *req;

    if (unlikely(csk->state == CTP_ABORTING || !skb))
        return;
    cxgbi_sock_set_state(csk, CTP_ABORTING);
    cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
    /* Purge the send queue so we don't send anything after an abort. */
    cxgbi_sock_purge_write_queue(csk);

    csk->cpl_abort_req = NULL;
    req = (struct cpl_abort_req *)skb->head;
    skb->priority = CPL_PRIORITY_DATA;
    set_arp_failure_handler(skb, abort_arp_failure);
    req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
    req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
    OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
    req->rsvd0 = htonl(csk->snd_nxt);
    req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
    req->cmd = CPL_ABORT_SEND_RST;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
        csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
        req->rsvd1);

    l2t_send(csk->cdev->lldev, skb, csk->l2t);
}

/*
 * CPL connection abort reply: host ->
 *
 * Send an ABORT_RPL message in response of the ABORT_REQ received.
 */
static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
{
    struct sk_buff *skb = csk->cpl_abort_rpl;
    struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u, status %d.\n",
        csk, csk->state, csk->flags, csk->tid, rst_status);

    csk->cpl_abort_rpl = NULL;
    skb->priority = CPL_PRIORITY_DATA;
    rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
    rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
    OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
    rpl->cmd = rst_status;
    cxgb3_ofld_send(csk->cdev->lldev, skb);
}

/*
 * CPL connection rx data ack: host ->
 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
 * credits sent.
 */
static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
{
    struct sk_buff *skb;
    struct cpl_rx_data_ack *req;
    u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
        "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
        csk, csk->state, csk->flags, csk->tid, credits, dack);

    skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
    if (!skb) {
        pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
        return 0;
    }
    req = (struct cpl_rx_data_ack *)skb->head;
    req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
    OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
    req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
                V_RX_CREDITS(credits));
    skb->priority = CPL_PRIORITY_ACK;
    cxgb3_ofld_send(csk->cdev->lldev, skb);
    return credits;
}

/*
 * CPL connection tx data: host ->
 *
 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
 * credits sent.
 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
 */

static unsigned int wrlen __read_mostly;
static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;

static void init_wr_tab(unsigned int wr_len)
{
    int i;

    if (skb_wrs[1])     /* already initialized */
        return;
    for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
        int sgl_len = (3 * i) / 2 + (i & 1);

        sgl_len += 3;
        skb_wrs[i] = (sgl_len <= wr_len
                  ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
    }
    wrlen = wr_len * 8;
}

static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
                   int len, int req_completion)
{
    struct tx_data_wr *req;
    struct l2t_entry *l2t = csk->l2t;

    skb_reset_transport_header(skb);
    req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
    req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
            (req_completion ? F_WR_COMPL : 0));
    req->wr_lo = htonl(V_WR_TID(csk->tid));
    /* len includes the length of any HW ULP additions */
    req->len = htonl(len);
    /* V_TX_ULP_SUBMODE sets both the mode and submode */
    req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) |
               V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
    req->sndseq = htonl(csk->snd_nxt);
    req->param = htonl(V_TX_PORT(l2t->smt_idx));

    if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
        req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
                    V_TX_CPU_IDX(csk->rss_qid));
        /* sendbuffer is in units of 32KB. */
        req->param |= htonl(V_TX_SNDBUF(cxgb3i_snd_win >> 15));
        cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
    }
}

static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
{
    kfree_skb(skb);
}

static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
{
    int total_size = 0;
    struct sk_buff *skb;

    if (unlikely(csk->state < CTP_ESTABLISHED ||
        csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
            log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
                "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
                csk, csk->state, csk->flags, csk->tid);
        return 0;
    }

    while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
        int len = skb->len; /* length before skb_push */
        int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
        int wrs_needed = skb_wrs[frags];

        if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
            wrs_needed = 1;

        WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);

        if (csk->wr_cred < wrs_needed) {
            log_debug(1 << CXGBI_DBG_PDU_TX,
                "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
                csk, skb->len, skb->data_len, frags,
                wrs_needed, csk->wr_cred);
            break;
        }

        __skb_unlink(skb, &csk->write_queue);
        skb->priority = CPL_PRIORITY_DATA;
        skb->csum = wrs_needed; /* remember this until the WR_ACK */
        csk->wr_cred -= wrs_needed;
        csk->wr_una_cred += wrs_needed;
        cxgbi_sock_enqueue_wr(csk, skb);

        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
            "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
            "left %u, unack %u.\n",
            csk, skb->len, skb->data_len, frags, skb->csum,
            csk->wr_cred, csk->wr_una_cred);

        if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
            if ((req_completion &&
                csk->wr_una_cred == wrs_needed) ||
                 csk->wr_una_cred >= csk->wr_max_cred / 2) {
                req_completion = 1;
                csk->wr_una_cred = 0;
            }
            len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
            make_tx_data_wr(csk, skb, len, req_completion);
            csk->snd_nxt += len;
            cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
        }
        total_size += skb->truesize;
        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
            "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
            csk, csk->tid, skb);
        set_arp_failure_handler(skb, arp_failure_skb_discard);
        l2t_send(csk->cdev->lldev, skb, csk->l2t);
    }
    return total_size;
}

/*
 * Process a CPL_ACT_ESTABLISH message: -> host
 * Updates connection state from an active establish CPL message.  Runs with
 * the connection lock held.
 */

static inline void free_atid(struct cxgbi_sock *csk)
{
    if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
        cxgb3_free_atid(csk->cdev->lldev, csk->atid);
        cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
        cxgbi_sock_put(csk);
    }
}

static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
{
    struct cxgbi_sock *csk = ctx;
    struct cpl_act_establish *req = cplhdr(skb);
    unsigned int tid = GET_TID(req);
    unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
    u32 rcv_isn = ntohl(req->rcv_isn);  /* real RCV_ISN + 1 */

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
        atid, atid, csk, csk->state, csk->flags, rcv_isn);

    cxgbi_sock_get(csk);
    cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
    csk->tid = tid;
    cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);

    free_atid(csk);

    csk->rss_qid = G_QNUM(ntohs(skb->csum));

    spin_lock_bh(&csk->lock);
    if (csk->retry_timer.function) {
        del_timer(&csk->retry_timer);
        csk->retry_timer.function = NULL;
    }

    if (unlikely(csk->state != CTP_ACTIVE_OPEN))
        pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
            csk, csk->state, csk->flags, csk->tid);

    csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
    if (cxgb3i_rcv_win > (M_RCV_BUFSIZ << 10))
        csk->rcv_wup -= cxgb3i_rcv_win - (M_RCV_BUFSIZ << 10);

    cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));

    if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
        /* upper layer has requested closing */
        send_abort_req(csk);
    else {
        if (skb_queue_len(&csk->write_queue))
            push_tx_frames(csk, 1);
        cxgbi_conn_tx_open(csk);
    }

    spin_unlock_bh(&csk->lock);
    __kfree_skb(skb);
    return 0;
}

/*
 * Process a CPL_ACT_OPEN_RPL message: -> host
 * Handle active open failures.
 */
static int act_open_rpl_status_to_errno(int status)
{
    switch (status) {
    case CPL_ERR_CONN_RESET:
        return -ECONNREFUSED;
    case CPL_ERR_ARP_MISS:
        return -EHOSTUNREACH;
    case CPL_ERR_CONN_TIMEDOUT:
        return -ETIMEDOUT;
    case CPL_ERR_TCAM_FULL:
        return -ENOMEM;
    case CPL_ERR_CONN_EXIST:
        return -EADDRINUSE;
    default:
        return -EIO;
    }
}

static void act_open_retry_timer(unsigned long data)
{
    struct sk_buff *skb;
    struct cxgbi_sock *csk = (struct cxgbi_sock *)data;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u.\n",
        csk, csk->state, csk->flags, csk->tid);

    cxgbi_sock_get(csk);
    spin_lock_bh(&csk->lock);
    skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
    if (!skb)
        cxgbi_sock_fail_act_open(csk, -ENOMEM);
    else {
        skb->sk = (struct sock *)csk;
        set_arp_failure_handler(skb, act_open_arp_failure);
        send_act_open_req(csk, skb, csk->l2t);
    }
    spin_unlock_bh(&csk->lock);
    cxgbi_sock_put(csk);
}

static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
{
    struct cxgbi_sock *csk = ctx;
    struct cpl_act_open_rpl *rpl = cplhdr(skb);

    pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
        csk, csk->state, csk->flags, csk->atid, rpl->status,
        &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
        &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));

    if (rpl->status != CPL_ERR_TCAM_FULL &&
        rpl->status != CPL_ERR_CONN_EXIST &&
        rpl->status != CPL_ERR_ARP_MISS)
        cxgb3_queue_tid_release(tdev, GET_TID(rpl));

    cxgbi_sock_get(csk);
    spin_lock_bh(&csk->lock);
    if (rpl->status == CPL_ERR_CONN_EXIST &&
        csk->retry_timer.function != act_open_retry_timer) {
        csk->retry_timer.function = act_open_retry_timer;
        mod_timer(&csk->retry_timer, jiffies + HZ / 2);
    } else
        cxgbi_sock_fail_act_open(csk,
                act_open_rpl_status_to_errno(rpl->status));

    spin_unlock_bh(&csk->lock);
    cxgbi_sock_put(csk);
    __kfree_skb(skb);
    return 0;
}

/*
 * Process PEER_CLOSE CPL messages: -> host
 * Handle peer FIN.
 */
static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
{
    struct cxgbi_sock *csk = ctx;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u.\n",
        csk, csk->state, csk->flags, csk->tid);

    cxgbi_sock_rcv_peer_close(csk);
    __kfree_skb(skb);
    return 0;
}

/*
 * Process CLOSE_CONN_RPL CPL message: -> host
 * Process a peer ACK to our FIN.
 */
static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
                void *ctx)
{
    struct cxgbi_sock *csk = ctx;
    struct cpl_close_con_rpl *rpl = cplhdr(skb);

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
        csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));

    cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
    __kfree_skb(skb);
    return 0;
}

/*
 * Process ABORT_REQ_RSS CPL message: -> host
 * Process abort requests.  If we are waiting for an ABORT_RPL we ignore this
 * request except that we need to reply to it.
 */

static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
                 int *need_rst)
{
    switch (abort_reason) {
    case CPL_ERR_BAD_SYN: /* fall through */
    case CPL_ERR_CONN_RESET:
        return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
    case CPL_ERR_XMIT_TIMEDOUT:
    case CPL_ERR_PERSIST_TIMEDOUT:
    case CPL_ERR_FINWAIT2_TIMEDOUT:
    case CPL_ERR_KEEPALIVE_TIMEDOUT:
        return -ETIMEDOUT;
    default:
        return -EIO;
    }
}

static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
{
    const struct cpl_abort_req_rss *req = cplhdr(skb);
    struct cxgbi_sock *csk = ctx;
    int rst_status = CPL_ABORT_NO_RST;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u.\n",
        csk, csk->state, csk->flags, csk->tid);

    if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
        req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
        goto done;
    }

    cxgbi_sock_get(csk);
    spin_lock_bh(&csk->lock);

    if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
        cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
        cxgbi_sock_set_state(csk, CTP_ABORTING);
        goto out;
    }

    cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
    send_abort_rpl(csk, rst_status);

    if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
        csk->err = abort_status_to_errno(csk, req->status, &rst_status);
        cxgbi_sock_closed(csk);
    }

out:
    spin_unlock_bh(&csk->lock);
    cxgbi_sock_put(csk);
done:
    __kfree_skb(skb);
    return 0;
}

/*
 * Process ABORT_RPL_RSS CPL message: -> host
 * Process abort replies.  We only process these messages if we anticipate
 * them as the coordination between SW and HW in this area is somewhat lacking
 * and sometimes we get ABORT_RPLs after we are done with the connection that
 * originated the ABORT_REQ.
 */
static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
{
    struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
    struct cxgbi_sock *csk = ctx;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
        rpl->status, csk, csk ? csk->state : 0,
        csk ? csk->flags : 0UL);
    /*
     * Ignore replies to post-close aborts indicating that the abort was
     * requested too late.  These connections are terminated when we get
     * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
     * arrives the TID is either no longer used or it has been recycled.
     */
    if (rpl->status == CPL_ERR_ABORT_FAILED)
        goto rel_skb;
    /*
     * Sometimes we've already closed the connection, e.g., a post-close
     * abort races with ABORT_REQ_RSS, the latter frees the connection
     * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
     * but FW turns the ABORT_REQ into a regular one and so we get
     * ABORT_RPL_RSS with status 0 and no connection.
     */
    if (csk)
        cxgbi_sock_rcv_abort_rpl(csk);
rel_skb:
    __kfree_skb(skb);
    return 0;
}

/*
 * Process RX_ISCSI_HDR CPL message: -> host
 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
 * follow after the bhs.
 */
static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
{
    struct cxgbi_sock *csk = ctx;
    struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
    struct cpl_iscsi_hdr_norss data_cpl;
    struct cpl_rx_data_ddp_norss ddp_cpl;
    unsigned int hdr_len, data_len, status;
    unsigned int len;
    int err;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
        "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
        csk, csk->state, csk->flags, csk->tid, skb, skb->len);

    spin_lock_bh(&csk->lock);

    if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
            "csk 0x%p,%u,0x%lx,%u, bad state.\n",
            csk, csk->state, csk->flags, csk->tid);
        if (csk->state != CTP_ABORTING)
            goto abort_conn;
        else
            goto discard;
    }

    cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
    cxgbi_skcb_flags(skb) = 0;

    skb_reset_transport_header(skb);
    __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));

    len = hdr_len = ntohs(hdr_cpl->len);
    /* msg coalesce is off or not enough data received */
    if (skb->len <= hdr_len) {
        pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
            csk->cdev->ports[csk->port_id]->name, csk->tid,
            skb->len, hdr_len);
        goto abort_conn;
    }
    cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);

    err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
                sizeof(ddp_cpl));
    if (err < 0) {
        pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
            csk->cdev->ports[csk->port_id]->name, csk->tid,
            skb->len, sizeof(ddp_cpl), err);
        goto abort_conn;
    }

    cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
    cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
    cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
    status = ntohl(ddp_cpl.ddp_status);

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
        "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
        csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);

    if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
        cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
    if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
        cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
    if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
        cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);

    if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
        err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
        if (err < 0) {
            pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
                csk->cdev->ports[csk->port_id]->name,
                csk->tid, sizeof(data_cpl), skb->len, err);
            goto abort_conn;
        }
        data_len = ntohs(data_cpl.len);
        log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
            "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
            skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
        len += sizeof(data_cpl) + data_len;
    } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
        cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);

    csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
    __pskb_trim(skb, len);
    __skb_queue_tail(&csk->receive_queue, skb);
    cxgbi_conn_pdu_ready(csk);

    spin_unlock_bh(&csk->lock);
    return 0;

abort_conn:
    send_abort_req(csk);
discard:
    spin_unlock_bh(&csk->lock);
    __kfree_skb(skb);
    return 0;
}

/*
 * Process TX_DATA_ACK CPL messages: -> host
 * Process an acknowledgment of WR completion.  Advance snd_una and send the
 * next batch of work requests from the write queue.
 */
static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
{
    struct cxgbi_sock *csk = ctx;
    struct cpl_wr_ack *hdr = cplhdr(skb);

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
        "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
        csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));

    cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
    __kfree_skb(skb);
    return 0;
}

/*
 * for each connection, pre-allocate skbs needed for close/abort requests. So
 * that we can service the request right away.
 */
static int alloc_cpls(struct cxgbi_sock *csk)
{
    csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,
                    GFP_KERNEL);
    if (!csk->cpl_close)
        return -ENOMEM;
    csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,
                    GFP_KERNEL);
    if (!csk->cpl_abort_req)
        goto free_cpl_skbs;

    csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,
                    GFP_KERNEL);
    if (!csk->cpl_abort_rpl)
        goto free_cpl_skbs;

    return 0;

free_cpl_skbs:
    cxgbi_sock_free_cpl_skbs(csk);
    return -ENOMEM;
}

static void l2t_put(struct cxgbi_sock *csk)
{
    struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;

    if (csk->l2t) {
        l2t_release(t3dev, csk->l2t);
        csk->l2t = NULL;
        cxgbi_sock_put(csk);
    }
}

static void release_offload_resources(struct cxgbi_sock *csk)
{
    struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx,%u.\n",
        csk, csk->state, csk->flags, csk->tid);

    csk->rss_qid = 0;
    cxgbi_sock_free_cpl_skbs(csk);

    if (csk->wr_cred != csk->wr_max_cred) {
        cxgbi_sock_purge_wr_queue(csk);
        cxgbi_sock_reset_wr_list(csk);
    }
    l2t_put(csk);
    if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
        free_atid(csk);
    else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
        cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);
        cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
        cxgbi_sock_put(csk);
    }
    csk->dst = NULL;
    csk->cdev = NULL;
}

static void update_address(struct cxgbi_hba *chba)
{
    if (chba->ipv4addr) {
        if (chba->vdev &&
            chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {
            cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);
            cxgb3i_set_private_ipv4addr(chba->ndev, 0);
            pr_info("%s set %pI4.\n",
                chba->vdev->name, &chba->ipv4addr);
        } else if (chba->ipv4addr !=
                cxgb3i_get_private_ipv4addr(chba->ndev)) {
            cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);
            pr_info("%s set %pI4.\n",
                chba->ndev->name, &chba->ipv4addr);
        }
    } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {
        if (chba->vdev)
            cxgb3i_set_private_ipv4addr(chba->vdev, 0);
        cxgb3i_set_private_ipv4addr(chba->ndev, 0);
    }
}

static int init_act_open(struct cxgbi_sock *csk)
{
    struct dst_entry *dst = csk->dst;
    struct cxgbi_device *cdev = csk->cdev;
    struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
    struct net_device *ndev = cdev->ports[csk->port_id];
    struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
    struct sk_buff *skb = NULL;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);

    update_address(chba);
    if (chba->ipv4addr)
        csk->saddr.sin_addr.s_addr = chba->ipv4addr;

    csk->rss_qid = 0;
    csk->l2t = t3_l2t_get(t3dev, dst, ndev,
                  &csk->daddr.sin_addr.s_addr);
    if (!csk->l2t) {
        pr_err("NO l2t available.\n");
        return -EINVAL;
    }
    cxgbi_sock_get(csk);

    csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
    if (csk->atid < 0) {
        pr_err("NO atid available.\n");
        goto rel_resource;
    }
    cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
    cxgbi_sock_get(csk);

    skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
    if (!skb)
        goto rel_resource;
    skb->sk = (struct sock *)csk;
    set_arp_failure_handler(skb, act_open_arp_failure);

    csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
    csk->wr_una_cred = 0;
    csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
    cxgbi_sock_reset_wr_list(csk);
    csk->err = 0;

    log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
        "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
        csk, csk->state, csk->flags,
        &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
        &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));

    cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
    send_act_open_req(csk, skb, csk->l2t);
    return 0;

rel_resource:
    if (skb)
        __kfree_skb(skb);
    return -EINVAL;
}

cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
    [CPL_ACT_ESTABLISH] = do_act_establish,
    [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
    [CPL_PEER_CLOSE] = do_peer_close,
    [CPL_ABORT_REQ_RSS] = do_abort_req,
    [CPL_ABORT_RPL_RSS] = do_abort_rpl,
    [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
    [CPL_TX_DMA_ACK] = do_wr_ack,
    [CPL_ISCSI_HDR] = do_iscsi_hdr,
};

int cxgb3i_ofld_init(struct cxgbi_device *cdev)
{
    struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
    struct adap_ports port;
    struct ofld_page_info rx_page_info;
    unsigned int wr_len;
    int rc;

    if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
        t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
        t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
        pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
        return -EINVAL;
    }

    if (cxgb3i_max_connect > CXGBI_MAX_CONN)
        cxgb3i_max_connect = CXGBI_MAX_CONN;

    rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,
                    cxgb3i_max_connect);
    if (rc < 0)
        return rc;

    init_wr_tab(wr_len);
    cdev->csk_release_offload_resources = release_offload_resources;
    cdev->csk_push_tx_frames = push_tx_frames;
    cdev->csk_send_abort_req = send_abort_req;
    cdev->csk_send_close_req = send_close_req;
    cdev->csk_send_rx_credits = send_rx_credits;
    cdev->csk_alloc_cpls = alloc_cpls;
    cdev->csk_init_act_open = init_act_open;

    pr_info("cdev 0x%p, offload up, added.\n", cdev);
    return 0;
}

/*
 * functions to program the pagepod in h/w
 */
static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
{
    struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;

    memset(req, 0, sizeof(*req));

    req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
    req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
                   V_ULPTX_CMD(ULP_MEM_WRITE));
    req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
             V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
}

static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr,
            unsigned int idx, unsigned int npods,
                struct cxgbi_gather_list *gl)
{
    struct cxgbi_device *cdev = csk->cdev;
    struct cxgbi_ddp_info *ddp = cdev->ddp;
    unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
    int i;

    log_debug(1 << CXGBI_DBG_DDP,
        "csk 0x%p, idx %u, npods %u, gl 0x%p.\n",
        csk, idx, npods, gl);

    for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
        struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
                        PPOD_SIZE, 0, GFP_ATOMIC);

        if (!skb)
            return -ENOMEM;

        ulp_mem_io_set_hdr(skb, pm_addr);
        cxgbi_ddp_ppod_set((struct cxgbi_pagepod *)(skb->head +
                    sizeof(struct ulp_mem_io)),
                   hdr, gl, i * PPOD_PAGES_MAX);
        skb->priority = CPL_PRIORITY_CONTROL;
        cxgb3_ofld_send(cdev->lldev, skb);
    }
    return 0;
}

static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag,
              unsigned int idx, unsigned int npods)
{
    struct cxgbi_device *cdev = chba->cdev;
    struct cxgbi_ddp_info *ddp = cdev->ddp;
    unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
    int i;

    log_debug(1 << CXGBI_DBG_DDP,
        "cdev 0x%p, idx %u, npods %u, tag 0x%x.\n",
        cdev, idx, npods, tag);

    for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
        struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
                        PPOD_SIZE, 0, GFP_ATOMIC);

        if (!skb) {
            pr_err("tag 0x%x, 0x%x, %d/%u, skb OOM.\n",
                tag, idx, i, npods);
            continue;
        }
        ulp_mem_io_set_hdr(skb, pm_addr);
        skb->priority = CPL_PRIORITY_CONTROL;
        cxgb3_ofld_send(cdev->lldev, skb);
    }
}

static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
                       unsigned int tid, int pg_idx, bool reply)
{
    struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
                    GFP_KERNEL);
    struct cpl_set_tcb_field *req;
    u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;

    log_debug(1 << CXGBI_DBG_DDP,
        "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
    if (!skb)
        return -ENOMEM;

    /* set up ulp submode and page size */
    req = (struct cpl_set_tcb_field *)skb->head;
    req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
    OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
    req->reply = V_NO_REPLY(reply ? 0 : 1);
    req->cpu_idx = 0;
    req->word = htons(31);
    req->mask = cpu_to_be64(0xF0000000);
    req->val = cpu_to_be64(val << 28);
    skb->priority = CPL_PRIORITY_CONTROL;

    cxgb3_ofld_send(csk->cdev->lldev, skb);
    return 0;
}

static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
                 int hcrc, int dcrc, int reply)
{
    struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
                    GFP_KERNEL);
    struct cpl_set_tcb_field *req;
    u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);

    log_debug(1 << CXGBI_DBG_DDP,
        "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
    if (!skb)
        return -ENOMEM;

    /* set up ulp submode and page size */
    req = (struct cpl_set_tcb_field *)skb->head;
    req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
    OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
    req->reply = V_NO_REPLY(reply ? 0 : 1);
    req->cpu_idx = 0;
    req->word = htons(31);
    req->mask = cpu_to_be64(0x0F000000);
    req->val = cpu_to_be64(val << 24);
    skb->priority = CPL_PRIORITY_CONTROL;

    cxgb3_ofld_send(csk->cdev->lldev, skb);
    return 0;
}

static void t3_ddp_cleanup(struct cxgbi_device *cdev)
{
    struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;

    if (cxgbi_ddp_cleanup(cdev)) {
        pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev);
        tdev->ulp_iscsi = NULL;
    }
}

static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
{
    struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
    struct cxgbi_ddp_info *ddp = tdev->ulp_iscsi;
    struct ulp_iscsi_info uinfo;
    unsigned int pgsz_factor[4];
    int i, err;

    if (ddp) {
        kref_get(&ddp->refcnt);
        pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n",
            tdev, tdev->ulp_iscsi);
        cdev->ddp = ddp;
        return -EALREADY;
    }

    err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
    if (err < 0) {
        pr_err("%s, failed to get iscsi param err=%d.\n",
             tdev->name, err);
        return err;
    }

    err = cxgbi_ddp_init(cdev, uinfo.llimit, uinfo.ulimit,
            uinfo.max_txsz, uinfo.max_rxsz);
    if (err < 0)
        return err;

    ddp = cdev->ddp;

    uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
    cxgbi_ddp_page_size_factor(pgsz_factor);
    for (i = 0; i < 4; i++)
        uinfo.pgsz_factor[i] = pgsz_factor[i];
    uinfo.ulimit = uinfo.llimit + (ddp->nppods << PPOD_SIZE_SHIFT);

    err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
    if (err < 0) {
        pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n",
            tdev->name, err);
        cxgbi_ddp_cleanup(cdev);
        return err;
    }
    tdev->ulp_iscsi = ddp;

    cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
    cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
    cdev->csk_ddp_set = ddp_set_map;
    cdev->csk_ddp_clear = ddp_clear_map;

    pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
        "%u/%u.\n",
        tdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask,
        ddp->rsvd_tag_mask, ddp->max_txsz, uinfo.max_txsz,
        ddp->max_rxsz, uinfo.max_rxsz);
    return 0;
}

static void cxgb3i_dev_close(struct t3cdev *t3dev)
{
    struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);

    if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
        pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
        return;
    }

    cxgbi_device_unregister(cdev);
}

static void cxgb3i_dev_open(struct t3cdev *t3dev)
{
    struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
    struct adapter *adapter = tdev2adap(t3dev);
    int i, err;

    if (cdev) {
        pr_info("0x%p, updating.\n", cdev);
        return;
    }

    cdev = cxgbi_device_register(0, adapter->params.nports);
    if (!cdev) {
        pr_warn("device 0x%p register failed.\n", t3dev);
        return;
    }

    cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
    cdev->lldev = t3dev;
    cdev->pdev = adapter->pdev;
    cdev->ports = adapter->port;
    cdev->nports = adapter->params.nports;
    cdev->mtus = adapter->params.mtus;
    cdev->nmtus = NMTUS;
    cdev->snd_win = cxgb3i_snd_win;
    cdev->rcv_win = cxgb3i_rcv_win;
    cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
    cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
    cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
    cdev->dev_ddp_cleanup = t3_ddp_cleanup;
    cdev->itp = &cxgb3i_iscsi_transport;

    err = cxgb3i_ddp_init(cdev);
    if (err) {
        pr_info("0x%p ddp init failed\n", cdev);
        goto err_out;
    }

    err = cxgb3i_ofld_init(cdev);
    if (err) {
        pr_info("0x%p offload init failed\n", cdev);
        goto err_out;
    }

    err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
                &cxgb3i_host_template, cxgb3i_stt);
    if (err)
        goto err_out;

    for (i = 0; i < cdev->nports; i++)
        cdev->hbas[i]->ipv4addr =
            cxgb3i_get_private_ipv4addr(cdev->ports[i]);

    pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
        cdev, cdev ? cdev->flags : 0, t3dev, err);
    return;

err_out:
    cxgbi_device_unregister(cdev);
}

static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
{
    struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);

    log_debug(1 << CXGBI_DBG_TOE,
        "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
        t3dev, cdev, event, port);
    if (!cdev)
        return;

    switch (event) {
    case OFFLOAD_STATUS_DOWN:
        cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
        break;
    case OFFLOAD_STATUS_UP:
        cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
        break;
    }
}

static int __init cxgb3i_init_module(void)
{
    int rc;

    printk(KERN_INFO "%s", version);

    rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
    if (rc < 0)
        return rc;

    cxgb3_register_client(&t3_client);
    return 0;
}

static void __exit cxgb3i_exit_module(void)
{
    cxgb3_unregister_client(&t3_client);
    cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
    cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
}

module_init(cxgb3i_init_module);
module_exit(cxgb3i_exit_module);