Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
iwch_provider.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses. You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  * Redistribution and use in source and binary forms, with or
11  * without modification, are permitted provided that the following
12  * conditions are met:
13  *
14  * - Redistributions of source code must retain the above
15  * copyright notice, this list of conditions and the following
16  * disclaimer.
17  *
18  * - Redistributions in binary form must reproduce the above
19  * copyright notice, this list of conditions and the following
20  * disclaimer in the documentation and/or other materials
21  * provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #include <linux/module.h>
33 #include <linux/moduleparam.h>
34 #include <linux/device.h>
35 #include <linux/netdevice.h>
36 #include <linux/etherdevice.h>
37 #include <linux/delay.h>
38 #include <linux/errno.h>
39 #include <linux/list.h>
40 #include <linux/sched.h>
41 #include <linux/spinlock.h>
42 #include <linux/ethtool.h>
43 #include <linux/rtnetlink.h>
44 #include <linux/inetdevice.h>
45 #include <linux/slab.h>
46 
47 #include <asm/io.h>
48 #include <asm/irq.h>
49 #include <asm/byteorder.h>
50 
51 #include <rdma/iw_cm.h>
52 #include <rdma/ib_verbs.h>
53 #include <rdma/ib_smi.h>
54 #include <rdma/ib_umem.h>
55 #include <rdma/ib_user_verbs.h>
56 
57 #include "cxio_hal.h"
58 #include "iwch.h"
59 #include "iwch_provider.h"
60 #include "iwch_cm.h"
61 #include "iwch_user.h"
62 #include "common.h"
63 
64 static struct ib_ah *iwch_ah_create(struct ib_pd *pd,
65  struct ib_ah_attr *ah_attr)
66 {
67  return ERR_PTR(-ENOSYS);
68 }
69 
70 static int iwch_ah_destroy(struct ib_ah *ah)
71 {
72  return -ENOSYS;
73 }
74 
75 static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
76 {
77  return -ENOSYS;
78 }
79 
80 static int iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
81 {
82  return -ENOSYS;
83 }
84 
85 static int iwch_process_mad(struct ib_device *ibdev,
86  int mad_flags,
87  u8 port_num,
88  struct ib_wc *in_wc,
89  struct ib_grh *in_grh,
90  struct ib_mad *in_mad, struct ib_mad *out_mad)
91 {
92  return -ENOSYS;
93 }
94 
95 static int iwch_dealloc_ucontext(struct ib_ucontext *context)
96 {
97  struct iwch_dev *rhp = to_iwch_dev(context->device);
98  struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
99  struct iwch_mm_entry *mm, *tmp;
100 
101  PDBG("%s context %p\n", __func__, context);
102  list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
103  kfree(mm);
104  cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
105  kfree(ucontext);
106  return 0;
107 }
108 
109 static struct ib_ucontext *iwch_alloc_ucontext(struct ib_device *ibdev,
110  struct ib_udata *udata)
111 {
112  struct iwch_ucontext *context;
113  struct iwch_dev *rhp = to_iwch_dev(ibdev);
114 
115  PDBG("%s ibdev %p\n", __func__, ibdev);
116  context = kzalloc(sizeof(*context), GFP_KERNEL);
117  if (!context)
118  return ERR_PTR(-ENOMEM);
119  cxio_init_ucontext(&rhp->rdev, &context->uctx);
120  INIT_LIST_HEAD(&context->mmaps);
121  spin_lock_init(&context->mmap_lock);
122  return &context->ibucontext;
123 }
124 
125 static int iwch_destroy_cq(struct ib_cq *ib_cq)
126 {
127  struct iwch_cq *chp;
128 
129  PDBG("%s ib_cq %p\n", __func__, ib_cq);
130  chp = to_iwch_cq(ib_cq);
131 
132  remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
133  atomic_dec(&chp->refcnt);
134  wait_event(chp->wait, !atomic_read(&chp->refcnt));
135 
136  cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
137  kfree(chp);
138  return 0;
139 }
140 
141 static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, int entries, int vector,
142  struct ib_ucontext *ib_context,
143  struct ib_udata *udata)
144 {
145  struct iwch_dev *rhp;
146  struct iwch_cq *chp;
147  struct iwch_create_cq_resp uresp;
148  struct iwch_create_cq_req ureq;
149  struct iwch_ucontext *ucontext = NULL;
150  static int warned;
151  size_t resplen;
152 
153  PDBG("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
154  rhp = to_iwch_dev(ibdev);
155  chp = kzalloc(sizeof(*chp), GFP_KERNEL);
156  if (!chp)
157  return ERR_PTR(-ENOMEM);
158 
159  if (ib_context) {
160  ucontext = to_iwch_ucontext(ib_context);
161  if (!t3a_device(rhp)) {
162  if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
163  kfree(chp);
164  return ERR_PTR(-EFAULT);
165  }
166  chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;
167  }
168  }
169 
170  if (t3a_device(rhp)) {
171 
172  /*
173  * T3A: Add some fluff to handle extra CQEs inserted
174  * for various errors.
175  * Additional CQE possibilities:
176  * TERMINATE,
177  * incoming RDMA WRITE Failures
178  * incoming RDMA READ REQUEST FAILUREs
179  * NOTE: We cannot ensure the CQ won't overflow.
180  */
181  entries += 16;
182  }
183  entries = roundup_pow_of_two(entries);
184  chp->cq.size_log2 = ilog2(entries);
185 
186  if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
187  kfree(chp);
188  return ERR_PTR(-ENOMEM);
189  }
190  chp->rhp = rhp;
191  chp->ibcq.cqe = 1 << chp->cq.size_log2;
192  spin_lock_init(&chp->lock);
194  atomic_set(&chp->refcnt, 1);
195  init_waitqueue_head(&chp->wait);
196  if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
197  cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
198  kfree(chp);
199  return ERR_PTR(-ENOMEM);
200  }
201 
202  if (ucontext) {
203  struct iwch_mm_entry *mm;
204 
205  mm = kmalloc(sizeof *mm, GFP_KERNEL);
206  if (!mm) {
207  iwch_destroy_cq(&chp->ibcq);
208  return ERR_PTR(-ENOMEM);
209  }
210  uresp.cqid = chp->cq.cqid;
211  uresp.size_log2 = chp->cq.size_log2;
212  spin_lock(&ucontext->mmap_lock);
213  uresp.key = ucontext->key;
214  ucontext->key += PAGE_SIZE;
215  spin_unlock(&ucontext->mmap_lock);
216  mm->key = uresp.key;
217  mm->addr = virt_to_phys(chp->cq.queue);
218  if (udata->outlen < sizeof uresp) {
219  if (!warned++)
220  printk(KERN_WARNING MOD "Warning - "
221  "downlevel libcxgb3 (non-fatal).\n");
222  mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
223  sizeof(struct t3_cqe));
224  resplen = sizeof(struct iwch_create_cq_resp_v0);
225  } else {
226  mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
227  sizeof(struct t3_cqe));
228  uresp.memsize = mm->len;
229  resplen = sizeof uresp;
230  }
231  if (ib_copy_to_udata(udata, &uresp, resplen)) {
232  kfree(mm);
233  iwch_destroy_cq(&chp->ibcq);
234  return ERR_PTR(-EFAULT);
235  }
236  insert_mmap(ucontext, mm);
237  }
238  PDBG("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
239  chp->cq.cqid, chp, (1 << chp->cq.size_log2),
240  (unsigned long long) chp->cq.dma_addr);
241  return &chp->ibcq;
242 }
243 
244 static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
245 {
246 #ifdef notyet
247  struct iwch_cq *chp = to_iwch_cq(cq);
248  struct t3_cq oldcq, newcq;
249  int ret;
250 
251  PDBG("%s ib_cq %p cqe %d\n", __func__, cq, cqe);
252 
253  /* We don't downsize... */
254  if (cqe <= cq->cqe)
255  return 0;
256 
257  /* create new t3_cq with new size */
258  cqe = roundup_pow_of_two(cqe+1);
259  newcq.size_log2 = ilog2(cqe);
260 
261  /* Dont allow resize to less than the current wce count */
262  if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
263  return -ENOMEM;
264  }
265 
266  /* Quiesce all QPs using this CQ */
267  ret = iwch_quiesce_qps(chp);
268  if (ret) {
269  return ret;
270  }
271 
272  ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
273  if (ret) {
274  return ret;
275  }
276 
277  /* copy CQEs */
278  memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
279  sizeof(struct t3_cqe));
280 
281  /* old iwch_qp gets new t3_cq but keeps old cqid */
282  oldcq = chp->cq;
283  chp->cq = newcq;
284  chp->cq.cqid = oldcq.cqid;
285 
286  /* resize new t3_cq to update the HW context */
287  ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
288  if (ret) {
289  chp->cq = oldcq;
290  return ret;
291  }
292  chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
293 
294  /* destroy old t3_cq */
295  oldcq.cqid = newcq.cqid;
296  ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
297  if (ret) {
298  printk(KERN_ERR MOD "%s - cxio_destroy_cq failed %d\n",
299  __func__, ret);
300  }
301 
302  /* add user hooks here */
303 
304  /* resume qps */
305  ret = iwch_resume_qps(chp);
306  return ret;
307 #else
308  return -ENOSYS;
309 #endif
310 }
311 
312 static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
313 {
314  struct iwch_dev *rhp;
315  struct iwch_cq *chp;
316  enum t3_cq_opcode cq_op;
317  int err;
318  unsigned long flag;
319  u32 rptr;
320 
321  chp = to_iwch_cq(ibcq);
322  rhp = chp->rhp;
323  if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
324  cq_op = CQ_ARM_SE;
325  else
326  cq_op = CQ_ARM_AN;
327  if (chp->user_rptr_addr) {
328  if (get_user(rptr, chp->user_rptr_addr))
329  return -EFAULT;
330  spin_lock_irqsave(&chp->lock, flag);
331  chp->cq.rptr = rptr;
332  } else
333  spin_lock_irqsave(&chp->lock, flag);
334  PDBG("%s rptr 0x%x\n", __func__, chp->cq.rptr);
335  err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
336  spin_unlock_irqrestore(&chp->lock, flag);
337  if (err < 0)
338  printk(KERN_ERR MOD "Error %d rearming CQID 0x%x\n", err,
339  chp->cq.cqid);
340  if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
341  err = 0;
342  return err;
343 }
344 
345 static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
346 {
347  int len = vma->vm_end - vma->vm_start;
348  u32 key = vma->vm_pgoff << PAGE_SHIFT;
349  struct cxio_rdev *rdev_p;
350  int ret = 0;
351  struct iwch_mm_entry *mm;
352  struct iwch_ucontext *ucontext;
353  u64 addr;
354 
355  PDBG("%s pgoff 0x%lx key 0x%x len %d\n", __func__, vma->vm_pgoff,
356  key, len);
357 
358  if (vma->vm_start & (PAGE_SIZE-1)) {
359  return -EINVAL;
360  }
361 
362  rdev_p = &(to_iwch_dev(context->device)->rdev);
363  ucontext = to_iwch_ucontext(context);
364 
365  mm = remove_mmap(ucontext, key, len);
366  if (!mm)
367  return -EINVAL;
368  addr = mm->addr;
369  kfree(mm);
370 
371  if ((addr >= rdev_p->rnic_info.udbell_physbase) &&
372  (addr < (rdev_p->rnic_info.udbell_physbase +
373  rdev_p->rnic_info.udbell_len))) {
374 
375  /*
376  * Map T3 DB register.
377  */
378  if (vma->vm_flags & VM_READ) {
379  return -EPERM;
380  }
381 
383  vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
384  vma->vm_flags &= ~VM_MAYREAD;
385  ret = io_remap_pfn_range(vma, vma->vm_start,
386  addr >> PAGE_SHIFT,
387  len, vma->vm_page_prot);
388  } else {
389 
390  /*
391  * Map WQ or CQ contig dma memory...
392  */
393  ret = remap_pfn_range(vma, vma->vm_start,
394  addr >> PAGE_SHIFT,
395  len, vma->vm_page_prot);
396  }
397 
398  return ret;
399 }
400 
401 static int iwch_deallocate_pd(struct ib_pd *pd)
402 {
403  struct iwch_dev *rhp;
404  struct iwch_pd *php;
405 
406  php = to_iwch_pd(pd);
407  rhp = php->rhp;
408  PDBG("%s ibpd %p pdid 0x%x\n", __func__, pd, php->pdid);
409  cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
410  kfree(php);
411  return 0;
412 }
413 
414 static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
415  struct ib_ucontext *context,
416  struct ib_udata *udata)
417 {
418  struct iwch_pd *php;
419  u32 pdid;
420  struct iwch_dev *rhp;
421 
422  PDBG("%s ibdev %p\n", __func__, ibdev);
423  rhp = (struct iwch_dev *) ibdev;
424  pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
425  if (!pdid)
426  return ERR_PTR(-EINVAL);
427  php = kzalloc(sizeof(*php), GFP_KERNEL);
428  if (!php) {
429  cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
430  return ERR_PTR(-ENOMEM);
431  }
432  php->pdid = pdid;
433  php->rhp = rhp;
434  if (context) {
435  if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) {
436  iwch_deallocate_pd(&php->ibpd);
437  return ERR_PTR(-EFAULT);
438  }
439  }
440  PDBG("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);
441  return &php->ibpd;
442 }
443 
444 static int iwch_dereg_mr(struct ib_mr *ib_mr)
445 {
446  struct iwch_dev *rhp;
447  struct iwch_mr *mhp;
448  u32 mmid;
449 
450  PDBG("%s ib_mr %p\n", __func__, ib_mr);
451  /* There can be no memory windows */
452  if (atomic_read(&ib_mr->usecnt))
453  return -EINVAL;
454 
455  mhp = to_iwch_mr(ib_mr);
456  rhp = mhp->rhp;
457  mmid = mhp->attr.stag >> 8;
458  cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
459  mhp->attr.pbl_addr);
460  iwch_free_pbl(mhp);
461  remove_handle(rhp, &rhp->mmidr, mmid);
462  if (mhp->kva)
463  kfree((void *) (unsigned long) mhp->kva);
464  if (mhp->umem)
465  ib_umem_release(mhp->umem);
466  PDBG("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
467  kfree(mhp);
468  return 0;
469 }
470 
471 static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
472  struct ib_phys_buf *buffer_list,
473  int num_phys_buf,
474  int acc,
475  u64 *iova_start)
476 {
477  __be64 *page_list;
478  int shift;
479  u64 total_size;
480  int npages;
481  struct iwch_dev *rhp;
482  struct iwch_pd *php;
483  struct iwch_mr *mhp;
484  int ret;
485 
486  PDBG("%s ib_pd %p\n", __func__, pd);
487  php = to_iwch_pd(pd);
488  rhp = php->rhp;
489 
490  mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
491  if (!mhp)
492  return ERR_PTR(-ENOMEM);
493 
494  mhp->rhp = rhp;
495 
496  /* First check that we have enough alignment */
497  if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
498  ret = -EINVAL;
499  goto err;
500  }
501 
502  if (num_phys_buf > 1 &&
503  ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
504  ret = -EINVAL;
505  goto err;
506  }
507 
508  ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
509  &total_size, &npages, &shift, &page_list);
510  if (ret)
511  goto err;
512 
513  ret = iwch_alloc_pbl(mhp, npages);
514  if (ret) {
515  kfree(page_list);
516  goto err_pbl;
517  }
518 
519  ret = iwch_write_pbl(mhp, page_list, npages, 0);
520  kfree(page_list);
521  if (ret)
522  goto err_pbl;
523 
524  mhp->attr.pdid = php->pdid;
525  mhp->attr.zbva = 0;
526 
527  mhp->attr.perms = iwch_ib_to_tpt_access(acc);
528  mhp->attr.va_fbo = *iova_start;
529  mhp->attr.page_size = shift - 12;
530 
531  mhp->attr.len = (u32) total_size;
532  mhp->attr.pbl_size = npages;
533  ret = iwch_register_mem(rhp, php, mhp, shift);
534  if (ret)
535  goto err_pbl;
536 
537  return &mhp->ibmr;
538 
539 err_pbl:
540  iwch_free_pbl(mhp);
541 
542 err:
543  kfree(mhp);
544  return ERR_PTR(ret);
545 
546 }
547 
548 static int iwch_reregister_phys_mem(struct ib_mr *mr,
549  int mr_rereg_mask,
550  struct ib_pd *pd,
551  struct ib_phys_buf *buffer_list,
552  int num_phys_buf,
553  int acc, u64 * iova_start)
554 {
555 
556  struct iwch_mr mh, *mhp;
557  struct iwch_pd *php;
558  struct iwch_dev *rhp;
559  __be64 *page_list = NULL;
560  int shift = 0;
561  u64 total_size;
562  int npages;
563  int ret;
564 
565  PDBG("%s ib_mr %p ib_pd %p\n", __func__, mr, pd);
566 
567  /* There can be no memory windows */
568  if (atomic_read(&mr->usecnt))
569  return -EINVAL;
570 
571  mhp = to_iwch_mr(mr);
572  rhp = mhp->rhp;
573  php = to_iwch_pd(mr->pd);
574 
575  /* make sure we are on the same adapter */
576  if (rhp != php->rhp)
577  return -EINVAL;
578 
579  memcpy(&mh, mhp, sizeof *mhp);
580 
581  if (mr_rereg_mask & IB_MR_REREG_PD)
582  php = to_iwch_pd(pd);
583  if (mr_rereg_mask & IB_MR_REREG_ACCESS)
584  mh.attr.perms = iwch_ib_to_tpt_access(acc);
585  if (mr_rereg_mask & IB_MR_REREG_TRANS) {
586  ret = build_phys_page_list(buffer_list, num_phys_buf,
587  iova_start,
588  &total_size, &npages,
589  &shift, &page_list);
590  if (ret)
591  return ret;
592  }
593 
594  ret = iwch_reregister_mem(rhp, php, &mh, shift, npages);
595  kfree(page_list);
596  if (ret) {
597  return ret;
598  }
599  if (mr_rereg_mask & IB_MR_REREG_PD)
600  mhp->attr.pdid = php->pdid;
601  if (mr_rereg_mask & IB_MR_REREG_ACCESS)
602  mhp->attr.perms = iwch_ib_to_tpt_access(acc);
603  if (mr_rereg_mask & IB_MR_REREG_TRANS) {
604  mhp->attr.zbva = 0;
605  mhp->attr.va_fbo = *iova_start;
606  mhp->attr.page_size = shift - 12;
607  mhp->attr.len = (u32) total_size;
608  mhp->attr.pbl_size = npages;
609  }
610 
611  return 0;
612 }
613 
614 
615 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
616  u64 virt, int acc, struct ib_udata *udata)
617 {
618  __be64 *pages;
619  int shift, n, len;
620  int i, j, k;
621  int err = 0;
622  struct ib_umem_chunk *chunk;
623  struct iwch_dev *rhp;
624  struct iwch_pd *php;
625  struct iwch_mr *mhp;
626  struct iwch_reg_user_mr_resp uresp;
627 
628  PDBG("%s ib_pd %p\n", __func__, pd);
629 
630  php = to_iwch_pd(pd);
631  rhp = php->rhp;
632  mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
633  if (!mhp)
634  return ERR_PTR(-ENOMEM);
635 
636  mhp->rhp = rhp;
637 
638  mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
639  if (IS_ERR(mhp->umem)) {
640  err = PTR_ERR(mhp->umem);
641  kfree(mhp);
642  return ERR_PTR(err);
643  }
644 
645  shift = ffs(mhp->umem->page_size) - 1;
646 
647  n = 0;
648  list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
649  n += chunk->nents;
650 
651  err = iwch_alloc_pbl(mhp, n);
652  if (err)
653  goto err;
654 
655  pages = (__be64 *) __get_free_page(GFP_KERNEL);
656  if (!pages) {
657  err = -ENOMEM;
658  goto err_pbl;
659  }
660 
661  i = n = 0;
662 
663  list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
664  for (j = 0; j < chunk->nmap; ++j) {
665  len = sg_dma_len(&chunk->page_list[j]) >> shift;
666  for (k = 0; k < len; ++k) {
667  pages[i++] = cpu_to_be64(sg_dma_address(
668  &chunk->page_list[j]) +
669  mhp->umem->page_size * k);
670  if (i == PAGE_SIZE / sizeof *pages) {
671  err = iwch_write_pbl(mhp, pages, i, n);
672  if (err)
673  goto pbl_done;
674  n += i;
675  i = 0;
676  }
677  }
678  }
679 
680  if (i)
681  err = iwch_write_pbl(mhp, pages, i, n);
682 
683 pbl_done:
684  free_page((unsigned long) pages);
685  if (err)
686  goto err_pbl;
687 
688  mhp->attr.pdid = php->pdid;
689  mhp->attr.zbva = 0;
690  mhp->attr.perms = iwch_ib_to_tpt_access(acc);
691  mhp->attr.va_fbo = virt;
692  mhp->attr.page_size = shift - 12;
693  mhp->attr.len = (u32) length;
694 
695  err = iwch_register_mem(rhp, php, mhp, shift);
696  if (err)
697  goto err_pbl;
698 
699  if (udata && !t3a_device(rhp)) {
700  uresp.pbl_addr = (mhp->attr.pbl_addr -
701  rhp->rdev.rnic_info.pbl_base) >> 3;
702  PDBG("%s user resp pbl_addr 0x%x\n", __func__,
703  uresp.pbl_addr);
704 
705  if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
706  iwch_dereg_mr(&mhp->ibmr);
707  err = -EFAULT;
708  goto err;
709  }
710  }
711 
712  return &mhp->ibmr;
713 
714 err_pbl:
715  iwch_free_pbl(mhp);
716 
717 err:
718  ib_umem_release(mhp->umem);
719  kfree(mhp);
720  return ERR_PTR(err);
721 }
722 
723 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
724 {
725  struct ib_phys_buf bl;
726  u64 kva;
727  struct ib_mr *ibmr;
728 
729  PDBG("%s ib_pd %p\n", __func__, pd);
730 
731  /*
732  * T3 only supports 32 bits of size.
733  */
734  bl.size = 0xffffffff;
735  bl.addr = 0;
736  kva = 0;
737  ibmr = iwch_register_phys_mem(pd, &bl, 1, acc, &kva);
738  return ibmr;
739 }
740 
741 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd)
742 {
743  struct iwch_dev *rhp;
744  struct iwch_pd *php;
745  struct iwch_mw *mhp;
746  u32 mmid;
747  u32 stag = 0;
748  int ret;
749 
750  php = to_iwch_pd(pd);
751  rhp = php->rhp;
752  mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
753  if (!mhp)
754  return ERR_PTR(-ENOMEM);
755  ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
756  if (ret) {
757  kfree(mhp);
758  return ERR_PTR(ret);
759  }
760  mhp->rhp = rhp;
761  mhp->attr.pdid = php->pdid;
762  mhp->attr.type = TPT_MW;
763  mhp->attr.stag = stag;
764  mmid = (stag) >> 8;
765  mhp->ibmw.rkey = stag;
766  if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
767  cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
768  kfree(mhp);
769  return ERR_PTR(-ENOMEM);
770  }
771  PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
772  return &(mhp->ibmw);
773 }
774 
775 static int iwch_dealloc_mw(struct ib_mw *mw)
776 {
777  struct iwch_dev *rhp;
778  struct iwch_mw *mhp;
779  u32 mmid;
780 
781  mhp = to_iwch_mw(mw);
782  rhp = mhp->rhp;
783  mmid = (mw->rkey) >> 8;
784  cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
785  remove_handle(rhp, &rhp->mmidr, mmid);
786  kfree(mhp);
787  PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
788  return 0;
789 }
790 
791 static struct ib_mr *iwch_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth)
792 {
793  struct iwch_dev *rhp;
794  struct iwch_pd *php;
795  struct iwch_mr *mhp;
796  u32 mmid;
797  u32 stag = 0;
798  int ret = 0;
799 
800  php = to_iwch_pd(pd);
801  rhp = php->rhp;
802  mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
803  if (!mhp)
804  goto err;
805 
806  mhp->rhp = rhp;
807  ret = iwch_alloc_pbl(mhp, pbl_depth);
808  if (ret)
809  goto err1;
810  mhp->attr.pbl_size = pbl_depth;
811  ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,
812  mhp->attr.pbl_size, mhp->attr.pbl_addr);
813  if (ret)
814  goto err2;
815  mhp->attr.pdid = php->pdid;
816  mhp->attr.type = TPT_NON_SHARED_MR;
817  mhp->attr.stag = stag;
818  mhp->attr.state = 1;
819  mmid = (stag) >> 8;
820  mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
821  if (insert_handle(rhp, &rhp->mmidr, mhp, mmid))
822  goto err3;
823 
824  PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
825  return &(mhp->ibmr);
826 err3:
827  cxio_dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
828  mhp->attr.pbl_addr);
829 err2:
830  iwch_free_pbl(mhp);
831 err1:
832  kfree(mhp);
833 err:
834  return ERR_PTR(ret);
835 }
836 
837 static struct ib_fast_reg_page_list *iwch_alloc_fastreg_pbl(
838  struct ib_device *device,
839  int page_list_len)
840 {
842 
843  page_list = kmalloc(sizeof *page_list + page_list_len * sizeof(u64),
844  GFP_KERNEL);
845  if (!page_list)
846  return ERR_PTR(-ENOMEM);
847 
848  page_list->page_list = (u64 *)(page_list + 1);
849  page_list->max_page_list_len = page_list_len;
850 
851  return page_list;
852 }
853 
854 static void iwch_free_fastreg_pbl(struct ib_fast_reg_page_list *page_list)
855 {
856  kfree(page_list);
857 }
858 
859 static int iwch_destroy_qp(struct ib_qp *ib_qp)
860 {
861  struct iwch_dev *rhp;
862  struct iwch_qp *qhp;
863  struct iwch_qp_attributes attrs;
864  struct iwch_ucontext *ucontext;
865 
866  qhp = to_iwch_qp(ib_qp);
867  rhp = qhp->rhp;
868 
869  attrs.next_state = IWCH_QP_STATE_ERROR;
870  iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
871  wait_event(qhp->wait, !qhp->ep);
872 
873  remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
874 
875  atomic_dec(&qhp->refcnt);
876  wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
877 
878  ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
879  : NULL;
880  cxio_destroy_qp(&rhp->rdev, &qhp->wq,
881  ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
882 
883  PDBG("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__,
884  ib_qp, qhp->wq.qpid, qhp);
885  kfree(qhp);
886  return 0;
887 }
888 
889 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
890  struct ib_qp_init_attr *attrs,
891  struct ib_udata *udata)
892 {
893  struct iwch_dev *rhp;
894  struct iwch_qp *qhp;
895  struct iwch_pd *php;
896  struct iwch_cq *schp;
897  struct iwch_cq *rchp;
898  struct iwch_create_qp_resp uresp;
899  int wqsize, sqsize, rqsize;
900  struct iwch_ucontext *ucontext;
901 
902  PDBG("%s ib_pd %p\n", __func__, pd);
903  if (attrs->qp_type != IB_QPT_RC)
904  return ERR_PTR(-EINVAL);
905  php = to_iwch_pd(pd);
906  rhp = php->rhp;
907  schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
908  rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
909  if (!schp || !rchp)
910  return ERR_PTR(-EINVAL);
911 
912  /* The RQT size must be # of entries + 1 rounded up to a power of two */
913  rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
914  if (rqsize == attrs->cap.max_recv_wr)
915  rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
916 
917  /* T3 doesn't support RQT depth < 16 */
918  if (rqsize < 16)
919  rqsize = 16;
920 
921  if (rqsize > T3_MAX_RQ_SIZE)
922  return ERR_PTR(-EINVAL);
923 
924  if (attrs->cap.max_inline_data > T3_MAX_INLINE)
925  return ERR_PTR(-EINVAL);
926 
927  /*
928  * NOTE: The SQ and total WQ sizes don't need to be
929  * a power of two. However, all the code assumes
930  * they are. EG: Q_FREECNT() and friends.
931  */
932  sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
933  wqsize = roundup_pow_of_two(rqsize + sqsize);
934 
935  /*
936  * Kernel users need more wq space for fastreg WRs which can take
937  * 2 WR fragments.
938  */
939  ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
940  if (!ucontext && wqsize < (rqsize + (2 * sqsize)))
941  wqsize = roundup_pow_of_two(rqsize +
942  roundup_pow_of_two(attrs->cap.max_send_wr * 2));
943  PDBG("%s wqsize %d sqsize %d rqsize %d\n", __func__,
944  wqsize, sqsize, rqsize);
945  qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
946  if (!qhp)
947  return ERR_PTR(-ENOMEM);
948  qhp->wq.size_log2 = ilog2(wqsize);
949  qhp->wq.rq_size_log2 = ilog2(rqsize);
950  qhp->wq.sq_size_log2 = ilog2(sqsize);
951  if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
952  ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
953  kfree(qhp);
954  return ERR_PTR(-ENOMEM);
955  }
956 
957  attrs->cap.max_recv_wr = rqsize - 1;
958  attrs->cap.max_send_wr = sqsize;
959  attrs->cap.max_inline_data = T3_MAX_INLINE;
960 
961  qhp->rhp = rhp;
962  qhp->attr.pd = php->pdid;
963  qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
964  qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
965  qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
966  qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
967  qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
968  qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
969  qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
970  qhp->attr.state = IWCH_QP_STATE_IDLE;
971  qhp->attr.next_state = IWCH_QP_STATE_IDLE;
972 
973  /*
974  * XXX - These don't get passed in from the openib user
975  * at create time. The CM sets them via a QP modify.
976  * Need to fix... I think the CM should
977  */
978  qhp->attr.enable_rdma_read = 1;
979  qhp->attr.enable_rdma_write = 1;
980  qhp->attr.enable_bind = 1;
981  qhp->attr.max_ord = 1;
982  qhp->attr.max_ird = 1;
983 
984  spin_lock_init(&qhp->lock);
985  init_waitqueue_head(&qhp->wait);
986  atomic_set(&qhp->refcnt, 1);
987 
988  if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
989  cxio_destroy_qp(&rhp->rdev, &qhp->wq,
990  ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
991  kfree(qhp);
992  return ERR_PTR(-ENOMEM);
993  }
994 
995  if (udata) {
996 
997  struct iwch_mm_entry *mm1, *mm2;
998 
999  mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
1000  if (!mm1) {
1001  iwch_destroy_qp(&qhp->ibqp);
1002  return ERR_PTR(-ENOMEM);
1003  }
1004 
1005  mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
1006  if (!mm2) {
1007  kfree(mm1);
1008  iwch_destroy_qp(&qhp->ibqp);
1009  return ERR_PTR(-ENOMEM);
1010  }
1011 
1012  uresp.qpid = qhp->wq.qpid;
1013  uresp.size_log2 = qhp->wq.size_log2;
1014  uresp.sq_size_log2 = qhp->wq.sq_size_log2;
1015  uresp.rq_size_log2 = qhp->wq.rq_size_log2;
1016  spin_lock(&ucontext->mmap_lock);
1017  uresp.key = ucontext->key;
1018  ucontext->key += PAGE_SIZE;
1019  uresp.db_key = ucontext->key;
1020  ucontext->key += PAGE_SIZE;
1021  spin_unlock(&ucontext->mmap_lock);
1022  if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
1023  kfree(mm1);
1024  kfree(mm2);
1025  iwch_destroy_qp(&qhp->ibqp);
1026  return ERR_PTR(-EFAULT);
1027  }
1028  mm1->key = uresp.key;
1029  mm1->addr = virt_to_phys(qhp->wq.queue);
1030  mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
1031  insert_mmap(ucontext, mm1);
1032  mm2->key = uresp.db_key;
1033  mm2->addr = qhp->wq.udb & PAGE_MASK;
1034  mm2->len = PAGE_SIZE;
1035  insert_mmap(ucontext, mm2);
1036  }
1037  qhp->ibqp.qp_num = qhp->wq.qpid;
1038  init_timer(&(qhp->timer));
1039  PDBG("%s sq_num_entries %d, rq_num_entries %d "
1040  "qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
1041  __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
1042  qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr,
1043  1 << qhp->wq.size_log2, qhp->wq.rq_addr);
1044  return &qhp->ibqp;
1045 }
1046 
1047 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1048  int attr_mask, struct ib_udata *udata)
1049 {
1050  struct iwch_dev *rhp;
1051  struct iwch_qp *qhp;
1052  enum iwch_qp_attr_mask mask = 0;
1053  struct iwch_qp_attributes attrs;
1054 
1055  PDBG("%s ib_qp %p\n", __func__, ibqp);
1056 
1057  /* iwarp does not support the RTR state */
1058  if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
1059  attr_mask &= ~IB_QP_STATE;
1060 
1061  /* Make sure we still have something left to do */
1062  if (!attr_mask)
1063  return 0;
1064 
1065  memset(&attrs, 0, sizeof attrs);
1066  qhp = to_iwch_qp(ibqp);
1067  rhp = qhp->rhp;
1068 
1069  attrs.next_state = iwch_convert_state(attr->qp_state);
1070  attrs.enable_rdma_read = (attr->qp_access_flags &
1071  IB_ACCESS_REMOTE_READ) ? 1 : 0;
1072  attrs.enable_rdma_write = (attr->qp_access_flags &
1073  IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1074  attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
1075 
1076 
1077  mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
1078  mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
1082 
1083  return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
1084 }
1085 
1086 void iwch_qp_add_ref(struct ib_qp *qp)
1087 {
1088  PDBG("%s ib_qp %p\n", __func__, qp);
1089  atomic_inc(&(to_iwch_qp(qp)->refcnt));
1090 }
1091 
1092 void iwch_qp_rem_ref(struct ib_qp *qp)
1093 {
1094  PDBG("%s ib_qp %p\n", __func__, qp);
1095  if (atomic_dec_and_test(&(to_iwch_qp(qp)->refcnt)))
1096  wake_up(&(to_iwch_qp(qp)->wait));
1097 }
1098 
1099 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
1100 {
1101  PDBG("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
1102  return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
1103 }
1104 
1105 
1106 static int iwch_query_pkey(struct ib_device *ibdev,
1107  u8 port, u16 index, u16 * pkey)
1108 {
1109  PDBG("%s ibdev %p\n", __func__, ibdev);
1110  *pkey = 0;
1111  return 0;
1112 }
1113 
1114 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
1115  int index, union ib_gid *gid)
1116 {
1117  struct iwch_dev *dev;
1118 
1119  PDBG("%s ibdev %p, port %d, index %d, gid %p\n",
1120  __func__, ibdev, port, index, gid);
1121  dev = to_iwch_dev(ibdev);
1122  BUG_ON(port == 0 || port > 2);
1123  memset(&(gid->raw[0]), 0, sizeof(gid->raw));
1124  memcpy(&(gid->raw[0]), dev->rdev.port_info.lldevs[port-1]->dev_addr, 6);
1125  return 0;
1126 }
1127 
1128 static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)
1129 {
1130  struct ethtool_drvinfo info;
1131  struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1132  char *cp, *next;
1133  unsigned fw_maj, fw_min, fw_mic;
1134 
1135  lldev->ethtool_ops->get_drvinfo(lldev, &info);
1136 
1137  next = info.fw_version + 1;
1138  cp = strsep(&next, ".");
1139  sscanf(cp, "%i", &fw_maj);
1140  cp = strsep(&next, ".");
1141  sscanf(cp, "%i", &fw_min);
1142  cp = strsep(&next, ".");
1143  sscanf(cp, "%i", &fw_mic);
1144 
1145  return (((u64)fw_maj & 0xffff) << 32) | ((fw_min & 0xffff) << 16) |
1146  (fw_mic & 0xffff);
1147 }
1148 
1149 static int iwch_query_device(struct ib_device *ibdev,
1150  struct ib_device_attr *props)
1151 {
1152 
1153  struct iwch_dev *dev;
1154  PDBG("%s ibdev %p\n", __func__, ibdev);
1155 
1156  dev = to_iwch_dev(ibdev);
1157  memset(props, 0, sizeof *props);
1158  memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1159  props->hw_ver = dev->rdev.t3cdev_p->type;
1160  props->fw_ver = fw_vers_string_to_u64(dev);
1161  props->device_cap_flags = dev->device_cap_flags;
1162  props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1163  props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
1164  props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
1165  props->max_mr_size = dev->attr.max_mr_size;
1166  props->max_qp = dev->attr.max_qps;
1167  props->max_qp_wr = dev->attr.max_wrs;
1168  props->max_sge = dev->attr.max_sge_per_wr;
1169  props->max_sge_rd = 1;
1170  props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1171  props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1172  props->max_cq = dev->attr.max_cqs;
1173  props->max_cqe = dev->attr.max_cqes_per_cq;
1174  props->max_mr = dev->attr.max_mem_regs;
1175  props->max_pd = dev->attr.max_pds;
1176  props->local_ca_ack_delay = 0;
1178 
1179  return 0;
1180 }
1181 
1182 static int iwch_query_port(struct ib_device *ibdev,
1183  u8 port, struct ib_port_attr *props)
1184 {
1185  struct iwch_dev *dev;
1186  struct net_device *netdev;
1187  struct in_device *inetdev;
1188 
1189  PDBG("%s ibdev %p\n", __func__, ibdev);
1190 
1191  dev = to_iwch_dev(ibdev);
1192  netdev = dev->rdev.port_info.lldevs[port-1];
1193 
1194  memset(props, 0, sizeof(struct ib_port_attr));
1195  props->max_mtu = IB_MTU_4096;
1196  if (netdev->mtu >= 4096)
1197  props->active_mtu = IB_MTU_4096;
1198  else if (netdev->mtu >= 2048)
1199  props->active_mtu = IB_MTU_2048;
1200  else if (netdev->mtu >= 1024)
1201  props->active_mtu = IB_MTU_1024;
1202  else if (netdev->mtu >= 512)
1203  props->active_mtu = IB_MTU_512;
1204  else
1205  props->active_mtu = IB_MTU_256;
1206 
1207  if (!netif_carrier_ok(netdev))
1208  props->state = IB_PORT_DOWN;
1209  else {
1210  inetdev = in_dev_get(netdev);
1211  if (inetdev) {
1212  if (inetdev->ifa_list)
1213  props->state = IB_PORT_ACTIVE;
1214  else
1215  props->state = IB_PORT_INIT;
1216  in_dev_put(inetdev);
1217  } else
1218  props->state = IB_PORT_INIT;
1219  }
1220 
1221  props->port_cap_flags =
1222  IB_PORT_CM_SUP |
1227  props->gid_tbl_len = 1;
1228  props->pkey_tbl_len = 1;
1229  props->active_width = 2;
1230  props->active_speed = IB_SPEED_DDR;
1231  props->max_msg_sz = -1;
1232 
1233  return 0;
1234 }
1235 
1236 static ssize_t show_rev(struct device *dev, struct device_attribute *attr,
1237  char *buf)
1238 {
1239  struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1240  ibdev.dev);
1241  PDBG("%s dev 0x%p\n", __func__, dev);
1242  return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);
1243 }
1244 
1245 static ssize_t show_fw_ver(struct device *dev, struct device_attribute *attr, char *buf)
1246 {
1247  struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1248  ibdev.dev);
1249  struct ethtool_drvinfo info;
1250  struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1251 
1252  PDBG("%s dev 0x%p\n", __func__, dev);
1253  lldev->ethtool_ops->get_drvinfo(lldev, &info);
1254  return sprintf(buf, "%s\n", info.fw_version);
1255 }
1256 
1257 static ssize_t show_hca(struct device *dev, struct device_attribute *attr,
1258  char *buf)
1259 {
1260  struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1261  ibdev.dev);
1262  struct ethtool_drvinfo info;
1263  struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1264 
1265  PDBG("%s dev 0x%p\n", __func__, dev);
1266  lldev->ethtool_ops->get_drvinfo(lldev, &info);
1267  return sprintf(buf, "%s\n", info.driver);
1268 }
1269 
1270 static ssize_t show_board(struct device *dev, struct device_attribute *attr,
1271  char *buf)
1272 {
1273  struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1274  ibdev.dev);
1275  PDBG("%s dev 0x%p\n", __func__, dev);
1276  return sprintf(buf, "%x.%x\n", iwch_dev->rdev.rnic_info.pdev->vendor,
1277  iwch_dev->rdev.rnic_info.pdev->device);
1278 }
1279 
1280 static int iwch_get_mib(struct ib_device *ibdev,
1281  union rdma_protocol_stats *stats)
1282 {
1283  struct iwch_dev *dev;
1284  struct tp_mib_stats m;
1285  int ret;
1286 
1287  PDBG("%s ibdev %p\n", __func__, ibdev);
1288  dev = to_iwch_dev(ibdev);
1289  ret = dev->rdev.t3cdev_p->ctl(dev->rdev.t3cdev_p, RDMA_GET_MIB, &m);
1290  if (ret)
1291  return -ENOSYS;
1292 
1293  memset(stats, 0, sizeof *stats);
1294  stats->iw.ipInReceives = ((u64) m.ipInReceive_hi << 32) +
1295  m.ipInReceive_lo;
1296  stats->iw.ipInHdrErrors = ((u64) m.ipInHdrErrors_hi << 32) +
1297  m.ipInHdrErrors_lo;
1298  stats->iw.ipInAddrErrors = ((u64) m.ipInAddrErrors_hi << 32) +
1299  m.ipInAddrErrors_lo;
1300  stats->iw.ipInUnknownProtos = ((u64) m.ipInUnknownProtos_hi << 32) +
1301  m.ipInUnknownProtos_lo;
1302  stats->iw.ipInDiscards = ((u64) m.ipInDiscards_hi << 32) +
1303  m.ipInDiscards_lo;
1304  stats->iw.ipInDelivers = ((u64) m.ipInDelivers_hi << 32) +
1305  m.ipInDelivers_lo;
1306  stats->iw.ipOutRequests = ((u64) m.ipOutRequests_hi << 32) +
1307  m.ipOutRequests_lo;
1308  stats->iw.ipOutDiscards = ((u64) m.ipOutDiscards_hi << 32) +
1309  m.ipOutDiscards_lo;
1310  stats->iw.ipOutNoRoutes = ((u64) m.ipOutNoRoutes_hi << 32) +
1311  m.ipOutNoRoutes_lo;
1312  stats->iw.ipReasmTimeout = (u64) m.ipReasmTimeout;
1313  stats->iw.ipReasmReqds = (u64) m.ipReasmReqds;
1314  stats->iw.ipReasmOKs = (u64) m.ipReasmOKs;
1315  stats->iw.ipReasmFails = (u64) m.ipReasmFails;
1316  stats->iw.tcpActiveOpens = (u64) m.tcpActiveOpens;
1317  stats->iw.tcpPassiveOpens = (u64) m.tcpPassiveOpens;
1318  stats->iw.tcpAttemptFails = (u64) m.tcpAttemptFails;
1319  stats->iw.tcpEstabResets = (u64) m.tcpEstabResets;
1320  stats->iw.tcpOutRsts = (u64) m.tcpOutRsts;
1321  stats->iw.tcpCurrEstab = (u64) m.tcpCurrEstab;
1322  stats->iw.tcpInSegs = ((u64) m.tcpInSegs_hi << 32) +
1323  m.tcpInSegs_lo;
1324  stats->iw.tcpOutSegs = ((u64) m.tcpOutSegs_hi << 32) +
1325  m.tcpOutSegs_lo;
1326  stats->iw.tcpRetransSegs = ((u64) m.tcpRetransSeg_hi << 32) +
1327  m.tcpRetransSeg_lo;
1328  stats->iw.tcpInErrs = ((u64) m.tcpInErrs_hi << 32) +
1329  m.tcpInErrs_lo;
1330  stats->iw.tcpRtoMin = (u64) m.tcpRtoMin;
1331  stats->iw.tcpRtoMax = (u64) m.tcpRtoMax;
1332  return 0;
1333 }
1334 
1335 static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
1336 static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
1337 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
1338 static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
1339 
1340 static struct device_attribute *iwch_class_attributes[] = {
1341  &dev_attr_hw_rev,
1342  &dev_attr_fw_ver,
1343  &dev_attr_hca_type,
1344  &dev_attr_board_id,
1345 };
1346 
1347 int iwch_register_device(struct iwch_dev *dev)
1348 {
1349  int ret;
1350  int i;
1351 
1352  PDBG("%s iwch_dev %p\n", __func__, dev);
1353  strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
1354  memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1355  memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1356  dev->ibdev.owner = THIS_MODULE;
1360 
1361  /* cxgb3 supports STag 0. */
1362  dev->ibdev.local_dma_lkey = 0;
1363 
1364  dev->ibdev.uverbs_cmd_mask =
1365  (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1366  (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1367  (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1368  (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1369  (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1370  (1ull << IB_USER_VERBS_CMD_REG_MR) |
1371  (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1373  (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1374  (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1376  (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1377  (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1378  (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1379  (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1380  (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1381  (1ull << IB_USER_VERBS_CMD_POST_RECV);
1382  dev->ibdev.node_type = RDMA_NODE_RNIC;
1383  memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1384  dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
1385  dev->ibdev.num_comp_vectors = 1;
1386  dev->ibdev.dma_device = &(dev->rdev.rnic_info.pdev->dev);
1387  dev->ibdev.query_device = iwch_query_device;
1388  dev->ibdev.query_port = iwch_query_port;
1389  dev->ibdev.query_pkey = iwch_query_pkey;
1390  dev->ibdev.query_gid = iwch_query_gid;
1391  dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
1392  dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
1393  dev->ibdev.mmap = iwch_mmap;
1394  dev->ibdev.alloc_pd = iwch_allocate_pd;
1395  dev->ibdev.dealloc_pd = iwch_deallocate_pd;
1396  dev->ibdev.create_ah = iwch_ah_create;
1397  dev->ibdev.destroy_ah = iwch_ah_destroy;
1398  dev->ibdev.create_qp = iwch_create_qp;
1399  dev->ibdev.modify_qp = iwch_ib_modify_qp;
1400  dev->ibdev.destroy_qp = iwch_destroy_qp;
1401  dev->ibdev.create_cq = iwch_create_cq;
1402  dev->ibdev.destroy_cq = iwch_destroy_cq;
1403  dev->ibdev.resize_cq = iwch_resize_cq;
1404  dev->ibdev.poll_cq = iwch_poll_cq;
1405  dev->ibdev.get_dma_mr = iwch_get_dma_mr;
1406  dev->ibdev.reg_phys_mr = iwch_register_phys_mem;
1407  dev->ibdev.rereg_phys_mr = iwch_reregister_phys_mem;
1408  dev->ibdev.reg_user_mr = iwch_reg_user_mr;
1409  dev->ibdev.dereg_mr = iwch_dereg_mr;
1410  dev->ibdev.alloc_mw = iwch_alloc_mw;
1411  dev->ibdev.bind_mw = iwch_bind_mw;
1412  dev->ibdev.dealloc_mw = iwch_dealloc_mw;
1413  dev->ibdev.alloc_fast_reg_mr = iwch_alloc_fast_reg_mr;
1414  dev->ibdev.alloc_fast_reg_page_list = iwch_alloc_fastreg_pbl;
1415  dev->ibdev.free_fast_reg_page_list = iwch_free_fastreg_pbl;
1416  dev->ibdev.attach_mcast = iwch_multicast_attach;
1417  dev->ibdev.detach_mcast = iwch_multicast_detach;
1418  dev->ibdev.process_mad = iwch_process_mad;
1419  dev->ibdev.req_notify_cq = iwch_arm_cq;
1420  dev->ibdev.post_send = iwch_post_send;
1421  dev->ibdev.post_recv = iwch_post_receive;
1422  dev->ibdev.get_protocol_stats = iwch_get_mib;
1423  dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1424 
1425  dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
1426  if (!dev->ibdev.iwcm)
1427  return -ENOMEM;
1428 
1429  dev->ibdev.iwcm->connect = iwch_connect;
1430  dev->ibdev.iwcm->accept = iwch_accept_cr;
1431  dev->ibdev.iwcm->reject = iwch_reject_cr;
1432  dev->ibdev.iwcm->create_listen = iwch_create_listen;
1433  dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1434  dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1435  dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1436  dev->ibdev.iwcm->get_qp = iwch_get_qp;
1437 
1438  ret = ib_register_device(&dev->ibdev, NULL);
1439  if (ret)
1440  goto bail1;
1441 
1442  for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i) {
1443  ret = device_create_file(&dev->ibdev.dev,
1444  iwch_class_attributes[i]);
1445  if (ret) {
1446  goto bail2;
1447  }
1448  }
1449  return 0;
1450 bail2:
1451  ib_unregister_device(&dev->ibdev);
1452 bail1:
1453  kfree(dev->ibdev.iwcm);
1454  return ret;
1455 }
1456 
1457 void iwch_unregister_device(struct iwch_dev *dev)
1458 {
1459  int i;
1460 
1461  PDBG("%s iwch_dev %p\n", __func__, dev);
1462  for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i)
1463  device_remove_file(&dev->ibdev.dev,
1464  iwch_class_attributes[i]);
1465  ib_unregister_device(&dev->ibdev);
1466  kfree(dev->ibdev.iwcm);
1467  return;
1468 }