Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
cxio_hal.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses. You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  * Redistribution and use in source and binary forms, with or
11  * without modification, are permitted provided that the following
12  * conditions are met:
13  *
14  * - Redistributions of source code must retain the above
15  * copyright notice, this list of conditions and the following
16  * disclaimer.
17  *
18  * - Redistributions in binary form must reproduce the above
19  * copyright notice, this list of conditions and the following
20  * disclaimer in the documentation and/or other materials
21  * provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #include <asm/delay.h>
33 
34 #include <linux/mutex.h>
35 #include <linux/netdevice.h>
36 #include <linux/sched.h>
37 #include <linux/spinlock.h>
38 #include <linux/pci.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/slab.h>
41 #include <net/net_namespace.h>
42 
43 #include "cxio_resource.h"
44 #include "cxio_hal.h"
45 #include "cxgb3_offload.h"
46 #include "sge_defs.h"
47 
48 static LIST_HEAD(rdev_list);
49 static cxio_hal_ev_callback_func_t cxio_ev_cb = NULL;
50 
51 static struct cxio_rdev *cxio_hal_find_rdev_by_name(char *dev_name)
52 {
53  struct cxio_rdev *rdev;
54 
55  list_for_each_entry(rdev, &rdev_list, entry)
56  if (!strcmp(rdev->dev_name, dev_name))
57  return rdev;
58  return NULL;
59 }
60 
61 static struct cxio_rdev *cxio_hal_find_rdev_by_t3cdev(struct t3cdev *tdev)
62 {
63  struct cxio_rdev *rdev;
64 
65  list_for_each_entry(rdev, &rdev_list, entry)
66  if (rdev->t3cdev_p == tdev)
67  return rdev;
68  return NULL;
69 }
70 
72  enum t3_cq_opcode op, u32 credit)
73 {
74  int ret;
75  struct t3_cqe *cqe;
76  u32 rptr;
77 
78  struct rdma_cq_op setup;
79  setup.id = cq->cqid;
80  setup.credits = (op == CQ_CREDIT_UPDATE) ? credit : 0;
81  setup.op = op;
82  ret = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_OP, &setup);
83 
84  if ((ret < 0) || (op == CQ_CREDIT_UPDATE))
85  return ret;
86 
87  /*
88  * If the rearm returned an index other than our current index,
89  * then there might be CQE's in flight (being DMA'd). We must wait
90  * here for them to complete or the consumer can miss a notification.
91  */
92  if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) {
93  int i=0;
94 
95  rptr = cq->rptr;
96 
97  /*
98  * Keep the generation correct by bumping rptr until it
99  * matches the index returned by the rearm - 1.
100  */
101  while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret)
102  rptr++;
103 
104  /*
105  * Now rptr is the index for the (last) cqe that was
106  * in-flight at the time the HW rearmed the CQ. We
107  * spin until that CQE is valid.
108  */
109  cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2);
110  while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) {
111  udelay(1);
112  if (i++ > 1000000) {
113  printk(KERN_ERR "%s: stalled rnic\n",
114  rdev_p->dev_name);
115  return -EIO;
116  }
117  }
118 
119  return 1;
120  }
121 
122  return 0;
123 }
124 
125 static int cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)
126 {
127  struct rdma_cq_setup setup;
128  setup.id = cqid;
129  setup.base_addr = 0; /* NULL address */
130  setup.size = 0; /* disaable the CQ */
131  setup.credits = 0;
132  setup.credit_thres = 0;
133  setup.ovfl_mode = 0;
134  return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
135 }
136 
137 static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
138 {
139  u64 sge_cmd;
140  struct t3_modify_qp_wr *wqe;
141  struct sk_buff *skb = alloc_skb(sizeof(*wqe), GFP_KERNEL);
142  if (!skb) {
143  PDBG("%s alloc_skb failed\n", __func__);
144  return -ENOMEM;
145  }
146  wqe = (struct t3_modify_qp_wr *) skb_put(skb, sizeof(*wqe));
147  memset(wqe, 0, sizeof(*wqe));
148  build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD,
149  T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 0, qpid, 7,
150  T3_SOPEOP);
152  sge_cmd = qpid << 8 | 3;
153  wqe->sge_cmd = cpu_to_be64(sge_cmd);
155  return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);
156 }
157 
158 int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel)
159 {
160  struct rdma_cq_setup setup;
161  int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe);
162 
163  size += 1; /* one extra page for storing cq-in-err state */
164  cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);
165  if (!cq->cqid)
166  return -ENOMEM;
167  if (kernel) {
168  cq->sw_queue = kzalloc(size, GFP_KERNEL);
169  if (!cq->sw_queue)
170  return -ENOMEM;
171  }
172  cq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev), size,
173  &(cq->dma_addr), GFP_KERNEL);
174  if (!cq->queue) {
175  kfree(cq->sw_queue);
176  return -ENOMEM;
177  }
179  memset(cq->queue, 0, size);
180  setup.id = cq->cqid;
181  setup.base_addr = (u64) (cq->dma_addr);
182  setup.size = 1UL << cq->size_log2;
183  setup.credits = 65535;
184  setup.credit_thres = 1;
185  if (rdev_p->t3cdev_p->type != T3A)
186  setup.ovfl_mode = 0;
187  else
188  setup.ovfl_mode = 1;
189  return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
190 }
191 
192 #ifdef notyet
193 int cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
194 {
195  struct rdma_cq_setup setup;
196  setup.id = cq->cqid;
197  setup.base_addr = (u64) (cq->dma_addr);
198  setup.size = 1UL << cq->size_log2;
199  setup.credits = setup.size;
200  setup.credit_thres = setup.size; /* TBD: overflow recovery */
201  setup.ovfl_mode = 1;
202  return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
203 }
204 #endif
205 
206 static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
207 {
208  struct cxio_qpid_list *entry;
209  u32 qpid;
210  int i;
211 
212  mutex_lock(&uctx->lock);
213  if (!list_empty(&uctx->qpids)) {
214  entry = list_entry(uctx->qpids.next, struct cxio_qpid_list,
215  entry);
216  list_del(&entry->entry);
217  qpid = entry->qpid;
218  kfree(entry);
219  } else {
220  qpid = cxio_hal_get_qpid(rdev_p->rscp);
221  if (!qpid)
222  goto out;
223  for (i = qpid+1; i & rdev_p->qpmask; i++) {
224  entry = kmalloc(sizeof *entry, GFP_KERNEL);
225  if (!entry)
226  break;
227  entry->qpid = i;
228  list_add_tail(&entry->entry, &uctx->qpids);
229  }
230  }
231 out:
232  mutex_unlock(&uctx->lock);
233  PDBG("%s qpid 0x%x\n", __func__, qpid);
234  return qpid;
235 }
236 
237 static void put_qpid(struct cxio_rdev *rdev_p, u32 qpid,
238  struct cxio_ucontext *uctx)
239 {
240  struct cxio_qpid_list *entry;
241 
242  entry = kmalloc(sizeof *entry, GFP_KERNEL);
243  if (!entry)
244  return;
245  PDBG("%s qpid 0x%x\n", __func__, qpid);
246  entry->qpid = qpid;
247  mutex_lock(&uctx->lock);
248  list_add_tail(&entry->entry, &uctx->qpids);
249  mutex_unlock(&uctx->lock);
250 }
251 
252 void cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
253 {
254  struct list_head *pos, *nxt;
255  struct cxio_qpid_list *entry;
256 
257  mutex_lock(&uctx->lock);
258  list_for_each_safe(pos, nxt, &uctx->qpids) {
259  entry = list_entry(pos, struct cxio_qpid_list, entry);
260  list_del_init(&entry->entry);
261  if (!(entry->qpid & rdev_p->qpmask))
262  cxio_hal_put_qpid(rdev_p->rscp, entry->qpid);
263  kfree(entry);
264  }
265  mutex_unlock(&uctx->lock);
266 }
267 
268 void cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
269 {
270  INIT_LIST_HEAD(&uctx->qpids);
271  mutex_init(&uctx->lock);
272 }
273 
274 int cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
275  struct t3_wq *wq, struct cxio_ucontext *uctx)
276 {
277  int depth = 1UL << wq->size_log2;
278  int rqsize = 1UL << wq->rq_size_log2;
279 
280  wq->qpid = get_qpid(rdev_p, uctx);
281  if (!wq->qpid)
282  return -ENOMEM;
283 
284  wq->rq = kzalloc(depth * sizeof(struct t3_swrq), GFP_KERNEL);
285  if (!wq->rq)
286  goto err1;
287 
288  wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);
289  if (!wq->rq_addr)
290  goto err2;
291 
292  wq->sq = kzalloc(depth * sizeof(struct t3_swsq), GFP_KERNEL);
293  if (!wq->sq)
294  goto err3;
295 
296  wq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev),
297  depth * sizeof(union t3_wr),
298  &(wq->dma_addr), GFP_KERNEL);
299  if (!wq->queue)
300  goto err4;
301 
302  memset(wq->queue, 0, depth * sizeof(union t3_wr));
304  wq->doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr;
305  if (!kernel_domain)
306  wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +
307  (wq->qpid << rdev_p->qpshift);
308  wq->rdev = rdev_p;
309  PDBG("%s qpid 0x%x doorbell 0x%p udb 0x%llx\n", __func__,
310  wq->qpid, wq->doorbell, (unsigned long long) wq->udb);
311  return 0;
312 err4:
313  kfree(wq->sq);
314 err3:
315  cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);
316 err2:
317  kfree(wq->rq);
318 err1:
319  put_qpid(rdev_p, wq->qpid, uctx);
320  return -ENOMEM;
321 }
322 
323 int cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
324 {
325  int err;
326  err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
327  kfree(cq->sw_queue);
328  dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),
329  (1UL << (cq->size_log2))
330  * sizeof(struct t3_cqe), cq->queue,
331  dma_unmap_addr(cq, mapping));
332  cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);
333  return err;
334 }
335 
336 int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,
337  struct cxio_ucontext *uctx)
338 {
339  dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),
340  (1UL << (wq->size_log2))
341  * sizeof(union t3_wr), wq->queue,
342  dma_unmap_addr(wq, mapping));
343  kfree(wq->sq);
344  cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));
345  kfree(wq->rq);
346  put_qpid(rdev_p, wq->qpid, uctx);
347  return 0;
348 }
349 
350 static void insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
351 {
352  struct t3_cqe cqe;
353 
354  PDBG("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __func__,
355  wq, cq, cq->sw_rptr, cq->sw_wptr);
356  memset(&cqe, 0, sizeof(cqe));
359  V_CQE_TYPE(0) |
360  V_CQE_SWCQE(1) |
361  V_CQE_QPID(wq->qpid) |
363  cq->size_log2)));
364  *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
365  cq->sw_wptr++;
366 }
367 
368 int cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
369 {
370  u32 ptr;
371  int flushed = 0;
372 
373  PDBG("%s wq %p cq %p\n", __func__, wq, cq);
374 
375  /* flush RQ */
376  PDBG("%s rq_rptr %u rq_wptr %u skip count %u\n", __func__,
377  wq->rq_rptr, wq->rq_wptr, count);
378  ptr = wq->rq_rptr + count;
379  while (ptr++ != wq->rq_wptr) {
380  insert_recv_cqe(wq, cq);
381  flushed++;
382  }
383  return flushed;
384 }
385 
386 static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
387  struct t3_swsq *sqp)
388 {
389  struct t3_cqe cqe;
390 
391  PDBG("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __func__,
392  wq, cq, cq->sw_rptr, cq->sw_wptr);
393  memset(&cqe, 0, sizeof(cqe));
395  V_CQE_OPCODE(sqp->opcode) |
396  V_CQE_TYPE(1) |
397  V_CQE_SWCQE(1) |
398  V_CQE_QPID(wq->qpid) |
400  cq->size_log2)));
401  cqe.u.scqe.wrid_hi = sqp->sq_wptr;
402 
403  *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
404  cq->sw_wptr++;
405 }
406 
407 int cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
408 {
409  __u32 ptr;
410  int flushed = 0;
411  struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2);
412 
413  ptr = wq->sq_rptr + count;
414  sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
415  while (ptr != wq->sq_wptr) {
416  sqp->signaled = 0;
417  insert_sq_cqe(wq, cq, sqp);
418  ptr++;
419  sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
420  flushed++;
421  }
422  return flushed;
423 }
424 
425 /*
426  * Move all CQEs from the HWCQ into the SWCQ.
427  */
428 void cxio_flush_hw_cq(struct t3_cq *cq)
429 {
430  struct t3_cqe *cqe, *swcqe;
431 
432  PDBG("%s cq %p cqid 0x%x\n", __func__, cq, cq->cqid);
433  cqe = cxio_next_hw_cqe(cq);
434  while (cqe) {
435  PDBG("%s flushing hwcq rptr 0x%x to swcq wptr 0x%x\n",
436  __func__, cq->rptr, cq->sw_wptr);
437  swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
438  *swcqe = *cqe;
439  swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1));
440  cq->sw_wptr++;
441  cq->rptr++;
442  cqe = cxio_next_hw_cqe(cq);
443  }
444 }
445 
446 static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq)
447 {
448  if (CQE_OPCODE(*cqe) == T3_TERMINATE)
449  return 0;
450 
451  if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) && RQ_TYPE(*cqe))
452  return 0;
453 
454  if ((CQE_OPCODE(*cqe) == T3_READ_RESP) && SQ_TYPE(*cqe))
455  return 0;
456 
457  if (CQE_SEND_OPCODE(*cqe) && RQ_TYPE(*cqe) &&
458  Q_EMPTY(wq->rq_rptr, wq->rq_wptr))
459  return 0;
460 
461  return 1;
462 }
463 
464 void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
465 {
466  struct t3_cqe *cqe;
467  u32 ptr;
468 
469  *count = 0;
470  ptr = cq->sw_rptr;
471  while (!Q_EMPTY(ptr, cq->sw_wptr)) {
472  cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
473  if ((SQ_TYPE(*cqe) ||
474  ((CQE_OPCODE(*cqe) == T3_READ_RESP) && wq->oldest_read)) &&
475  (CQE_QPID(*cqe) == wq->qpid))
476  (*count)++;
477  ptr++;
478  }
479  PDBG("%s cq %p count %d\n", __func__, cq, *count);
480 }
481 
482 void cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
483 {
484  struct t3_cqe *cqe;
485  u32 ptr;
486 
487  *count = 0;
488  PDBG("%s count zero %d\n", __func__, *count);
489  ptr = cq->sw_rptr;
490  while (!Q_EMPTY(ptr, cq->sw_wptr)) {
491  cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
492  if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) &&
493  (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq))
494  (*count)++;
495  ptr++;
496  }
497  PDBG("%s cq %p count %d\n", __func__, cq, *count);
498 }
499 
500 static int cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)
501 {
502  struct rdma_cq_setup setup;
503  setup.id = 0;
504  setup.base_addr = 0; /* NULL address */
505  setup.size = 1; /* enable the CQ */
506  setup.credits = 0;
507 
508  /* force SGE to redirect to RspQ and interrupt */
509  setup.credit_thres = 0;
510  setup.ovfl_mode = 1;
511  return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
512 }
513 
514 static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
515 {
516  int err;
517  u64 sge_cmd, ctx0, ctx1;
518  u64 base_addr;
519  struct t3_modify_qp_wr *wqe;
520  struct sk_buff *skb;
521 
522  skb = alloc_skb(sizeof(*wqe), GFP_KERNEL);
523  if (!skb) {
524  PDBG("%s alloc_skb failed\n", __func__);
525  return -ENOMEM;
526  }
527  err = cxio_hal_init_ctrl_cq(rdev_p);
528  if (err) {
529  PDBG("%s err %d initializing ctrl_cq\n", __func__, err);
530  goto err;
531  }
532  rdev_p->ctrl_qp.workq = dma_alloc_coherent(
533  &(rdev_p->rnic_info.pdev->dev),
534  (1 << T3_CTRL_QP_SIZE_LOG2) *
535  sizeof(union t3_wr),
536  &(rdev_p->ctrl_qp.dma_addr),
537  GFP_KERNEL);
538  if (!rdev_p->ctrl_qp.workq) {
539  PDBG("%s dma_alloc_coherent failed\n", __func__);
540  err = -ENOMEM;
541  goto err;
542  }
544  rdev_p->ctrl_qp.dma_addr);
545  rdev_p->ctrl_qp.doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr;
546  memset(rdev_p->ctrl_qp.workq, 0,
547  (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr));
548 
549  mutex_init(&rdev_p->ctrl_qp.lock);
550  init_waitqueue_head(&rdev_p->ctrl_qp.waitq);
551 
552  /* update HW Ctrl QP context */
553  base_addr = rdev_p->ctrl_qp.dma_addr;
554  base_addr >>= 12;
555  ctx0 = (V_EC_SIZE((1 << T3_CTRL_QP_SIZE_LOG2)) |
556  V_EC_BASE_LO((u32) base_addr & 0xffff));
557  ctx0 <<= 32;
558  ctx0 |= V_EC_CREDITS(FW_WR_NUM);
559  base_addr >>= 16;
560  ctx1 = (u32) base_addr;
561  base_addr >>= 32;
562  ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr & 0xf) | V_EC_RESPQ(0) |
563  V_EC_TYPE(0) | V_EC_GEN(1) |
565  wqe = (struct t3_modify_qp_wr *) skb_put(skb, sizeof(*wqe));
566  memset(wqe, 0, sizeof(*wqe));
567  build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 0, 0,
570  sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;
571  wqe->sge_cmd = cpu_to_be64(sge_cmd);
572  wqe->ctx1 = cpu_to_be64(ctx1);
573  wqe->ctx0 = cpu_to_be64(ctx0);
574  PDBG("CtrlQP dma_addr 0x%llx workq %p size %d\n",
575  (unsigned long long) rdev_p->ctrl_qp.dma_addr,
576  rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
578  return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);
579 err:
580  kfree_skb(skb);
581  return err;
582 }
583 
584 static int cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)
585 {
586  dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),
588  * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,
589  dma_unmap_addr(&rdev_p->ctrl_qp, mapping));
590  return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);
591 }
592 
593 /* write len bytes of data into addr (32B aligned address)
594  * If data is NULL, clear len byte of memory to zero.
595  * caller acquires the ctrl_qp lock before the call
596  */
597 static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
598  u32 len, void *data)
599 {
600  u32 i, nr_wqe, copy_len;
601  u8 *copy_data;
602  u8 wr_len, utx_len; /* length in 8 byte flit */
603  enum t3_wr_flags flag;
604  __be64 *wqe;
605  u64 utx_cmd;
606  addr &= 0x7FFFFFF;
607  nr_wqe = len % 96 ? len / 96 + 1 : len / 96; /* 96B max per WQE */
608  PDBG("%s wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x\n",
609  __func__, rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,
610  nr_wqe, data, addr);
611  utx_len = 3; /* in 32B unit */
612  for (i = 0; i < nr_wqe; i++) {
613  if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,
615  PDBG("%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, "
616  "wait for more space i %d\n", __func__,
617  rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);
618  if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,
619  !Q_FULL(rdev_p->ctrl_qp.rptr,
620  rdev_p->ctrl_qp.wptr,
622  PDBG("%s ctrl_qp workq interrupted\n",
623  __func__);
624  return -ERESTARTSYS;
625  }
626  PDBG("%s ctrl_qp wakeup, continue posting work request "
627  "i %d\n", __func__, i);
628  }
629  wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
630  (1 << T3_CTRL_QP_SIZE_LOG2)));
631  flag = 0;
632  if (i == (nr_wqe - 1)) {
633  /* last WQE */
634  flag = T3_COMPLETION_FLAG;
635  if (len % 32)
636  utx_len = len / 32 + 1;
637  else
638  utx_len = len / 32;
639  }
640 
641  /*
642  * Force a CQE to return the credit to the workq in case
643  * we posted more than half the max QP size of WRs
644  */
645  if ((i != 0) &&
646  (i % (((1 << T3_CTRL_QP_SIZE_LOG2)) >> 1) == 0)) {
647  flag = T3_COMPLETION_FLAG;
648  PDBG("%s force completion at i %d\n", __func__, i);
649  }
650 
651  /* build the utx mem command */
652  wqe += (sizeof(struct t3_bypass_wr) >> 3);
653  utx_cmd = (T3_UTX_MEM_WRITE << 28) | (addr + i * 3);
654  utx_cmd <<= 32;
655  utx_cmd |= (utx_len << 28) | ((utx_len << 2) + 1);
656  *wqe = cpu_to_be64(utx_cmd);
657  wqe++;
658  copy_data = (u8 *) data + i * 96;
659  copy_len = len > 96 ? 96 : len;
660 
661  /* clear memory content if data is NULL */
662  if (data)
663  memcpy(wqe, copy_data, copy_len);
664  else
665  memset(wqe, 0, copy_len);
666  if (copy_len % 32)
667  memset(((u8 *) wqe) + copy_len, 0,
668  32 - (copy_len % 32));
669  wr_len = ((sizeof(struct t3_bypass_wr)) >> 3) + 1 +
670  (utx_len << 2);
671  wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
672  (1 << T3_CTRL_QP_SIZE_LOG2)));
673 
674  /* wptr in the WRID[31:0] */
675  ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;
676 
677  /*
678  * This must be the last write with a memory barrier
679  * for the genbit
680  */
681  build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,
682  Q_GENBIT(rdev_p->ctrl_qp.wptr,
684  wr_len, T3_SOPEOP);
685  if (flag == T3_COMPLETION_FLAG)
686  ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);
687  len -= 96;
688  rdev_p->ctrl_qp.wptr++;
689  }
690  return 0;
691 }
692 
693 /* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl_size and pbl_addr
694  * OUT: stag index
695  * TBD: shared memory region support
696  */
697 static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
698  u32 *stag, u8 stag_state, u32 pdid,
699  enum tpt_mem_type type, enum tpt_mem_perm perm,
700  u32 zbva, u64 to, u32 len, u8 page_size,
701  u32 pbl_size, u32 pbl_addr)
702 {
703  int err;
704  struct tpt_entry tpt;
705  u32 stag_idx;
706  u32 wptr;
707 
708  if (cxio_fatal_error(rdev_p))
709  return -EIO;
710 
711  stag_state = stag_state > 0;
712  stag_idx = (*stag) >> 8;
713 
714  if ((!reset_tpt_entry) && !(*stag != T3_STAG_UNSET)) {
715  stag_idx = cxio_hal_get_stag(rdev_p->rscp);
716  if (!stag_idx)
717  return -ENOMEM;
718  *stag = (stag_idx << 8) | ((*stag) & 0xFF);
719  }
720  PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
721  __func__, stag_state, type, pdid, stag_idx);
722 
723  mutex_lock(&rdev_p->ctrl_qp.lock);
724 
725  /* write TPT entry */
726  if (reset_tpt_entry)
727  memset(&tpt, 0, sizeof(tpt));
728  else {
729  tpt.valid_stag_pdid = cpu_to_be32(F_TPT_VALID |
730  V_TPT_STAG_KEY((*stag) & M_TPT_STAG_KEY) |
731  V_TPT_STAG_STATE(stag_state) |
732  V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid));
733  BUG_ON(page_size >= 28);
734  tpt.flags_pagesize_qpid = cpu_to_be32(V_TPT_PERM(perm) |
735  ((perm & TPT_MW_BIND) ? F_TPT_MW_BIND_ENABLE : 0) |
736  V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |
737  V_TPT_PAGE_SIZE(page_size));
738  tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 :
739  cpu_to_be32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, pbl_addr)>>3));
740  tpt.len = cpu_to_be32(len);
741  tpt.va_hi = cpu_to_be32((u32) (to >> 32));
742  tpt.va_low_or_fbo = cpu_to_be32((u32) (to & 0xFFFFFFFFULL));
743  tpt.rsvd_bind_cnt_or_pstag = 0;
744  tpt.rsvd_pbl_size = reset_tpt_entry ? 0 :
745  cpu_to_be32(V_TPT_PBL_SIZE(pbl_size >> 2));
746  }
747  err = cxio_hal_ctrl_qp_write_mem(rdev_p,
748  stag_idx +
749  (rdev_p->rnic_info.tpt_base >> 5),
750  sizeof(tpt), &tpt);
751 
752  /* release the stag index to free pool */
753  if (reset_tpt_entry)
754  cxio_hal_put_stag(rdev_p->rscp, stag_idx);
755 
756  wptr = rdev_p->ctrl_qp.wptr;
757  mutex_unlock(&rdev_p->ctrl_qp.lock);
758  if (!err)
759  if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,
760  SEQ32_GE(rdev_p->ctrl_qp.rptr,
761  wptr)))
762  return -ERESTARTSYS;
763  return err;
764 }
765 
766 int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,
767  u32 pbl_addr, u32 pbl_size)
768 {
769  u32 wptr;
770  int err;
771 
772  PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",
773  __func__, pbl_addr, rdev_p->rnic_info.pbl_base,
774  pbl_size);
775 
776  mutex_lock(&rdev_p->ctrl_qp.lock);
777  err = cxio_hal_ctrl_qp_write_mem(rdev_p, pbl_addr >> 5, pbl_size << 3,
778  pbl);
779  wptr = rdev_p->ctrl_qp.wptr;
780  mutex_unlock(&rdev_p->ctrl_qp.lock);
781  if (err)
782  return err;
783 
784  if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,
785  SEQ32_GE(rdev_p->ctrl_qp.rptr,
786  wptr)))
787  return -ERESTARTSYS;
788 
789  return 0;
790 }
791 
792 int cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
793  enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
794  u8 page_size, u32 pbl_size, u32 pbl_addr)
795 {
796  *stag = T3_STAG_UNSET;
797  return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
798  zbva, to, len, page_size, pbl_size, pbl_addr);
799 }
800 
801 int cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
802  enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
803  u8 page_size, u32 pbl_size, u32 pbl_addr)
804 {
805  return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
806  zbva, to, len, page_size, pbl_size, pbl_addr);
807 }
808 
809 int cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,
810  u32 pbl_addr)
811 {
812  return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,
813  pbl_size, pbl_addr);
814 }
815 
816 int cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)
817 {
818  *stag = T3_STAG_UNSET;
819  return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,
820  0, 0);
821 }
822 
823 int cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
824 {
825  return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,
826  0, 0);
827 }
828 
829 int cxio_allocate_stag(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr)
830 {
831  *stag = T3_STAG_UNSET;
832  return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_NON_SHARED_MR,
833  0, 0, 0ULL, 0, 0, pbl_size, pbl_addr);
834 }
835 
836 int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
837 {
838  struct t3_rdma_init_wr *wqe;
839  struct sk_buff *skb = alloc_skb(sizeof(*wqe), GFP_ATOMIC);
840  if (!skb)
841  return -ENOMEM;
842  PDBG("%s rdev_p %p\n", __func__, rdev_p);
843  wqe = (struct t3_rdma_init_wr *) __skb_put(skb, sizeof(*wqe));
844  wqe->wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_INIT));
845  wqe->wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(attr->tid) |
846  V_FW_RIWR_LEN(sizeof(*wqe) >> 3));
847  wqe->wrid.id1 = 0;
848  wqe->qpid = cpu_to_be32(attr->qpid);
849  wqe->pdid = cpu_to_be32(attr->pdid);
850  wqe->scqid = cpu_to_be32(attr->scqid);
851  wqe->rcqid = cpu_to_be32(attr->rcqid);
852  wqe->rq_addr = cpu_to_be32(attr->rq_addr - rdev_p->rnic_info.rqt_base);
853  wqe->rq_size = cpu_to_be32(attr->rq_size);
854  wqe->mpaattrs = attr->mpaattrs;
855  wqe->qpcaps = attr->qpcaps;
856  wqe->ulpdu_size = cpu_to_be16(attr->tcp_emss);
857  wqe->rqe_count = cpu_to_be16(attr->rqe_count);
858  wqe->flags_rtr_type = cpu_to_be16(attr->flags |
859  V_RTR_TYPE(attr->rtr_type) |
860  V_CHAN(attr->chan));
861  wqe->ord = cpu_to_be32(attr->ord);
862  wqe->ird = cpu_to_be32(attr->ird);
863  wqe->qp_dma_addr = cpu_to_be64(attr->qp_dma_addr);
864  wqe->qp_dma_size = cpu_to_be32(attr->qp_dma_size);
865  wqe->irs = cpu_to_be32(attr->irs);
866  skb->priority = 0; /* 0=>ToeQ; 1=>CtrlQ */
867  return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);
868 }
869 
871 {
872  cxio_ev_cb = ev_cb;
873 }
874 
876 {
877  cxio_ev_cb = NULL;
878 }
879 
880 static int cxio_hal_ev_handler(struct t3cdev *t3cdev_p, struct sk_buff *skb)
881 {
882  static int cnt;
883  struct cxio_rdev *rdev_p = NULL;
884  struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) skb->data;
885  PDBG("%d: %s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x"
886  " se %0x notify %0x cqbranch %0x creditth %0x\n",
887  cnt, __func__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),
888  RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg),
889  RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),
890  RSPQ_CREDIT_THRESH(rsp_msg));
891  PDBG("CQE: QPID 0x%0x genbit %0x type 0x%0x status 0x%0x opcode %d "
892  "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",
893  CQE_QPID(rsp_msg->cqe), CQE_GENBIT(rsp_msg->cqe),
894  CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
895  CQE_OPCODE(rsp_msg->cqe), CQE_LEN(rsp_msg->cqe),
896  CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
897  rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;
898  if (!rdev_p) {
899  PDBG("%s called by t3cdev %p with null ulp\n", __func__,
900  t3cdev_p);
901  return 0;
902  }
903  if (CQE_QPID(rsp_msg->cqe) == T3_CTRL_QP_ID) {
904  rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;
905  wake_up_interruptible(&rdev_p->ctrl_qp.waitq);
906  dev_kfree_skb_irq(skb);
907  } else if (CQE_QPID(rsp_msg->cqe) == 0xfff8)
908  dev_kfree_skb_irq(skb);
909  else if (cxio_ev_cb)
910  (*cxio_ev_cb) (rdev_p, skb);
911  else
912  dev_kfree_skb_irq(skb);
913  cnt++;
914  return 0;
915 }
916 
917 /* Caller takes care of locking if needed */
918 int cxio_rdev_open(struct cxio_rdev *rdev_p)
919 {
920  struct net_device *netdev_p = NULL;
921  int err = 0;
922  if (strlen(rdev_p->dev_name)) {
923  if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {
924  return -EBUSY;
925  }
926  netdev_p = dev_get_by_name(&init_net, rdev_p->dev_name);
927  if (!netdev_p) {
928  return -EINVAL;
929  }
930  dev_put(netdev_p);
931  } else if (rdev_p->t3cdev_p) {
932  if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p)) {
933  return -EBUSY;
934  }
935  netdev_p = rdev_p->t3cdev_p->lldev;
936  strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,
938  } else {
939  PDBG("%s t3cdev_p or dev_name must be set\n", __func__);
940  return -EINVAL;
941  }
942 
943  list_add_tail(&rdev_p->entry, &rdev_list);
944 
945  PDBG("%s opening rnic dev %s\n", __func__, rdev_p->dev_name);
946  memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));
947  if (!rdev_p->t3cdev_p)
948  rdev_p->t3cdev_p = dev2t3cdev(netdev_p);
949  rdev_p->t3cdev_p->ulp = (void *) rdev_p;
950 
951  err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_EMBEDDED_INFO,
952  &(rdev_p->fw_info));
953  if (err) {
954  printk(KERN_ERR "%s t3cdev_p(%p)->ctl returned error %d.\n",
955  __func__, rdev_p->t3cdev_p, err);
956  goto err1;
957  }
958  if (G_FW_VERSION_MAJOR(rdev_p->fw_info.fw_vers) != CXIO_FW_MAJ) {
959  printk(KERN_ERR MOD "fatal firmware version mismatch: "
960  "need version %u but adapter has version %u\n",
961  CXIO_FW_MAJ,
962  G_FW_VERSION_MAJOR(rdev_p->fw_info.fw_vers));
963  err = -EINVAL;
964  goto err1;
965  }
966 
967  err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,
968  &(rdev_p->rnic_info));
969  if (err) {
970  printk(KERN_ERR "%s t3cdev_p(%p)->ctl returned error %d.\n",
971  __func__, rdev_p->t3cdev_p, err);
972  goto err1;
973  }
974  err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,
975  &(rdev_p->port_info));
976  if (err) {
977  printk(KERN_ERR "%s t3cdev_p(%p)->ctl returned error %d.\n",
978  __func__, rdev_p->t3cdev_p, err);
979  goto err1;
980  }
981 
982  /*
983  * qpshift is the number of bits to shift the qpid left in order
984  * to get the correct address of the doorbell for that qp.
985  */
986  cxio_init_ucontext(rdev_p, &rdev_p->uctx);
987  rdev_p->qpshift = PAGE_SHIFT -
988  ilog2(65536 >>
989  ilog2(rdev_p->rnic_info.udbell_len >>
990  PAGE_SHIFT));
991  rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;
992  rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;
993  PDBG("%s rnic %s info: tpt_base 0x%0x tpt_top 0x%0x num stags %d "
994  "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x\n",
995  __func__, rdev_p->dev_name, rdev_p->rnic_info.tpt_base,
996  rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p),
997  rdev_p->rnic_info.pbl_base,
998  rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,
999  rdev_p->rnic_info.rqt_top);
1000  PDBG("udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu "
1001  "qpnr %d qpmask 0x%x\n",
1002  rdev_p->rnic_info.udbell_len,
1003  rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,
1004  rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);
1005 
1006  err = cxio_hal_init_ctrl_qp(rdev_p);
1007  if (err) {
1008  printk(KERN_ERR "%s error %d initializing ctrl_qp.\n",
1009  __func__, err);
1010  goto err1;
1011  }
1012  err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,
1014  T3_MAX_NUM_PD);
1015  if (err) {
1016  printk(KERN_ERR "%s error %d initializing hal resources.\n",
1017  __func__, err);
1018  goto err2;
1019  }
1020  err = cxio_hal_pblpool_create(rdev_p);
1021  if (err) {
1022  printk(KERN_ERR "%s error %d initializing pbl mem pool.\n",
1023  __func__, err);
1024  goto err3;
1025  }
1026  err = cxio_hal_rqtpool_create(rdev_p);
1027  if (err) {
1028  printk(KERN_ERR "%s error %d initializing rqt mem pool.\n",
1029  __func__, err);
1030  goto err4;
1031  }
1032  return 0;
1033 err4:
1034  cxio_hal_pblpool_destroy(rdev_p);
1035 err3:
1037 err2:
1038  cxio_hal_destroy_ctrl_qp(rdev_p);
1039 err1:
1040  rdev_p->t3cdev_p->ulp = NULL;
1041  list_del(&rdev_p->entry);
1042  return err;
1043 }
1044 
1045 void cxio_rdev_close(struct cxio_rdev *rdev_p)
1046 {
1047  if (rdev_p) {
1048  cxio_hal_pblpool_destroy(rdev_p);
1049  cxio_hal_rqtpool_destroy(rdev_p);
1050  list_del(&rdev_p->entry);
1051  cxio_hal_destroy_ctrl_qp(rdev_p);
1053  rdev_p->t3cdev_p->ulp = NULL;
1054  }
1055 }
1056 
1058 {
1060  return -ENOMEM;
1061  t3_register_cpl_handler(CPL_ASYNC_NOTIF, cxio_hal_ev_handler);
1062  return 0;
1063 }
1064 
1066 {
1067  struct cxio_rdev *rdev, *tmp;
1068 
1070  list_for_each_entry_safe(rdev, tmp, &rdev_list, entry)
1071  cxio_rdev_close(rdev);
1073 }
1074 
1075 static void flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
1076 {
1077  struct t3_swsq *sqp;
1078  __u32 ptr = wq->sq_rptr;
1079  int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
1080 
1081  sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1082  while (count--)
1083  if (!sqp->signaled) {
1084  ptr++;
1085  sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1086  } else if (sqp->complete) {
1087 
1088  /*
1089  * Insert this completed cqe into the swcq.
1090  */
1091  PDBG("%s moving cqe into swcq sq idx %ld cq idx %ld\n",
1092  __func__, Q_PTR2IDX(ptr, wq->sq_size_log2),
1093  Q_PTR2IDX(cq->sw_wptr, cq->size_log2));
1094  sqp->cqe.header |= htonl(V_CQE_SWCQE(1));
1095  *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))
1096  = sqp->cqe;
1097  cq->sw_wptr++;
1098  sqp->signaled = 0;
1099  break;
1100  } else
1101  break;
1102 }
1103 
1104 static void create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,
1105  struct t3_cqe *read_cqe)
1106 {
1107  read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr;
1108  read_cqe->len = wq->oldest_read->read_len;
1109  read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
1110  V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
1112  V_CQE_TYPE(1));
1113 }
1114 
1115 /*
1116  * Return a ptr to the next read wr in the SWSQ or NULL.
1117  */
1118 static void advance_oldest_read(struct t3_wq *wq)
1119 {
1120 
1121  u32 rptr = wq->oldest_read - wq->sq + 1;
1122  u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
1123 
1124  while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
1125  wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
1126 
1127  if (wq->oldest_read->opcode == T3_READ_REQ)
1128  return;
1129  rptr++;
1130  }
1131  wq->oldest_read = NULL;
1132 }
1133 
1134 /*
1135  * cxio_poll_cq
1136  *
1137  * Caller must:
1138  * check the validity of the first CQE,
1139  * supply the wq assicated with the qpid.
1140  *
1141  * credit: cq credit to return to sge.
1142  * cqe_flushed: 1 iff the CQE is flushed.
1143  * cqe: copy of the polled CQE.
1144  *
1145  * return value:
1146  * 0 CQE returned,
1147  * -1 CQE skipped, try again.
1148  */
1149 int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
1150  u8 *cqe_flushed, u64 *cookie, u32 *credit)
1151 {
1152  int ret = 0;
1153  struct t3_cqe *hw_cqe, read_cqe;
1154 
1155  *cqe_flushed = 0;
1156  *credit = 0;
1157  hw_cqe = cxio_next_cqe(cq);
1158 
1159  PDBG("%s CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x"
1160  " opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",
1161  __func__, CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),
1162  CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe),
1163  CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),
1164  CQE_WRID_LOW(*hw_cqe));
1165 
1166  /*
1167  * skip cqe's not affiliated with a QP.
1168  */
1169  if (wq == NULL) {
1170  ret = -1;
1171  goto skip_cqe;
1172  }
1173 
1174  /*
1175  * Gotta tweak READ completions:
1176  * 1) the cqe doesn't contain the sq_wptr from the wr.
1177  * 2) opcode not reflected from the wr.
1178  * 3) read_len not reflected from the wr.
1179  * 4) cq_type is RQ_TYPE not SQ_TYPE.
1180  */
1181  if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {
1182 
1183  /*
1184  * If this is an unsolicited read response, then the read
1185  * was generated by the kernel driver as part of peer-2-peer
1186  * connection setup. So ignore the completion.
1187  */
1188  if (!wq->oldest_read) {
1189  if (CQE_STATUS(*hw_cqe))
1190  wq->error = 1;
1191  ret = -1;
1192  goto skip_cqe;
1193  }
1194 
1195  /*
1196  * Don't write to the HWCQ, so create a new read req CQE
1197  * in local memory.
1198  */
1199  create_read_req_cqe(wq, hw_cqe, &read_cqe);
1200  hw_cqe = &read_cqe;
1201  advance_oldest_read(wq);
1202  }
1203 
1204  /*
1205  * T3A: Discard TERMINATE CQEs.
1206  */
1207  if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {
1208  ret = -1;
1209  wq->error = 1;
1210  goto skip_cqe;
1211  }
1212 
1213  if (CQE_STATUS(*hw_cqe) || wq->error) {
1214  *cqe_flushed = wq->error;
1215  wq->error = 1;
1216 
1217  /*
1218  * T3A inserts errors into the CQE. We cannot return
1219  * these as work completions.
1220  */
1221  /* incoming write failures */
1222  if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)
1223  && RQ_TYPE(*hw_cqe)) {
1224  ret = -1;
1225  goto skip_cqe;
1226  }
1227  /* incoming read request failures */
1228  if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) {
1229  ret = -1;
1230  goto skip_cqe;
1231  }
1232 
1233  /* incoming SEND with no receive posted failures */
1234  if (CQE_SEND_OPCODE(*hw_cqe) && RQ_TYPE(*hw_cqe) &&
1235  Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1236  ret = -1;
1237  goto skip_cqe;
1238  }
1239  BUG_ON((*cqe_flushed == 0) && !SW_CQE(*hw_cqe));
1240  goto proc_cqe;
1241  }
1242 
1243  /*
1244  * RECV completion.
1245  */
1246  if (RQ_TYPE(*hw_cqe)) {
1247 
1248  /*
1249  * HW only validates 4 bits of MSN. So we must validate that
1250  * the MSN in the SEND is the next expected MSN. If its not,
1251  * then we complete this with TPT_ERR_MSN and mark the wq in
1252  * error.
1253  */
1254 
1255  if (Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1256  wq->error = 1;
1257  ret = -1;
1258  goto skip_cqe;
1259  }
1260 
1261  if (unlikely((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) {
1262  wq->error = 1;
1263  hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));
1264  goto proc_cqe;
1265  }
1266  goto proc_cqe;
1267  }
1268 
1269  /*
1270  * If we get here its a send completion.
1271  *
1272  * Handle out of order completion. These get stuffed
1273  * in the SW SQ. Then the SW SQ is walked to move any
1274  * now in-order completions into the SW CQ. This handles
1275  * 2 cases:
1276  * 1) reaping unsignaled WRs when the first subsequent
1277  * signaled WR is completed.
1278  * 2) out of order read completions.
1279  */
1280  if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
1281  struct t3_swsq *sqp;
1282 
1283  PDBG("%s out of order completion going in swsq at idx %ld\n",
1284  __func__,
1285  Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2));
1286  sqp = wq->sq +
1287  Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
1288  sqp->cqe = *hw_cqe;
1289  sqp->complete = 1;
1290  ret = -1;
1291  goto flush_wq;
1292  }
1293 
1294 proc_cqe:
1295  *cqe = *hw_cqe;
1296 
1297  /*
1298  * Reap the associated WR(s) that are freed up with this
1299  * completion.
1300  */
1301  if (SQ_TYPE(*hw_cqe)) {
1302  wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
1303  PDBG("%s completing sq idx %ld\n", __func__,
1304  Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
1305  *cookie = wq->sq[Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2)].wr_id;
1306  wq->sq_rptr++;
1307  } else {
1308  PDBG("%s completing rq idx %ld\n", __func__,
1309  Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1310  *cookie = wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].wr_id;
1311  if (wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].pbl_addr)
1313  wq->rq[Q_PTR2IDX(wq->rq_rptr,
1314  wq->rq_size_log2)].pbl_addr, T3_STAG0_PBL_SIZE);
1315  BUG_ON(Q_EMPTY(wq->rq_rptr, wq->rq_wptr));
1316  wq->rq_rptr++;
1317  }
1318 
1319 flush_wq:
1320  /*
1321  * Flush any completed cqes that are now in-order.
1322  */
1323  flush_completed_wrs(wq, cq);
1324 
1325 skip_cqe:
1326  if (SW_CQE(*hw_cqe)) {
1327  PDBG("%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x\n",
1328  __func__, cq, cq->cqid, cq->sw_rptr);
1329  ++cq->sw_rptr;
1330  } else {
1331  PDBG("%s cq %p cqid 0x%x skip hw cqe rptr 0x%x\n",
1332  __func__, cq, cq->cqid, cq->rptr);
1333  ++cq->rptr;
1334 
1335  /*
1336  * T3A: compute credits.
1337  */
1338  if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
1339  || ((cq->rptr - cq->wptr) >= 128)) {
1340  *credit = cq->rptr - cq->wptr;
1341  cq->wptr = cq->rptr;
1342  }
1343  }
1344  return ret;
1345 }