Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
mthca_provider.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004 Topspin Communications. All rights reserved.
3  * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses. You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  * Redistribution and use in source and binary forms, with or
13  * without modification, are permitted provided that the following
14  * conditions are met:
15  *
16  * - Redistributions of source code must retain the above
17  * copyright notice, this list of conditions and the following
18  * disclaimer.
19  *
20  * - Redistributions in binary form must reproduce the above
21  * copyright notice, this list of conditions and the following
22  * disclaimer in the documentation and/or other materials
23  * provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 
35 #ifndef MTHCA_PROVIDER_H
36 #define MTHCA_PROVIDER_H
37 
38 #include <rdma/ib_verbs.h>
39 #include <rdma/ib_pack.h>
40 
41 #define MTHCA_MPT_FLAG_ATOMIC (1 << 14)
42 #define MTHCA_MPT_FLAG_REMOTE_WRITE (1 << 13)
43 #define MTHCA_MPT_FLAG_REMOTE_READ (1 << 12)
44 #define MTHCA_MPT_FLAG_LOCAL_WRITE (1 << 11)
45 #define MTHCA_MPT_FLAG_LOCAL_READ (1 << 10)
46 
48  void *buf;
50 };
51 
52 union mthca_buf {
55 };
56 
57 struct mthca_uar {
58  unsigned long pfn;
59  int index;
60 };
61 
62 struct mthca_user_db_table;
63 
66  struct mthca_uar uar;
69 };
70 
71 struct mthca_mtt;
72 
73 struct mthca_mr {
74  struct ib_mr ibmr;
75  struct ib_umem *umem;
76  struct mthca_mtt *mtt;
77 };
78 
79 struct mthca_fmr {
80  struct ib_fmr ibmr;
81  struct ib_fmr_attr attr;
82  struct mthca_mtt *mtt;
83  int maps;
84  union {
85  struct {
88  } tavor;
89  struct {
93  } arbel;
94  } mem;
95 };
96 
97 struct mthca_pd {
98  struct ib_pd ibpd;
101  struct mthca_mr ntmr;
103 };
104 
105 struct mthca_eq {
106  struct mthca_dev *dev;
107  int eqn;
112  int have_irq;
113  int nent;
115  struct mthca_mr mr;
117 };
118 
119 struct mthca_av;
120 
125 };
126 
127 struct mthca_ah {
128  struct ib_ah ibah;
131  struct mthca_av *av;
133 };
134 
135 /*
136  * Quick description of our CQ/QP locking scheme:
137  *
138  * We have one global lock that protects dev->cq/qp_table. Each
139  * struct mthca_cq/qp also has its own lock. An individual qp lock
140  * may be taken inside of an individual cq lock. Both cqs attached to
141  * a qp may be locked, with the cq with the lower cqn locked first.
142  * No other nesting should be done.
143  *
144  * Each struct mthca_cq/qp also has an ref count, protected by the
145  * corresponding table lock. The pointer from the cq/qp_table to the
146  * struct counts as one reference. This reference also is good for
147  * access through the consumer API, so modifying the CQ/QP etc doesn't
148  * need to take another reference. Access to a QP because of a
149  * completion being polled does not need a reference either.
150  *
151  * Finally, each struct mthca_cq/qp has a wait_queue_head_t for the
152  * destroy function to sleep on.
153  *
154  * This means that access from the consumer API requires nothing but
155  * taking the struct's lock.
156  *
157  * Access because of a completion event should go as follows:
158  * - lock cq/qp_table and look up struct
159  * - increment ref count in struct
160  * - drop cq/qp_table lock
161  * - lock struct, do your thing, and unlock struct
162  * - decrement ref count; if zero, wake up waiters
163  *
164  * To destroy a CQ/QP, we can do the following:
165  * - lock cq/qp_table
166  * - remove pointer and decrement ref count
167  * - unlock cq/qp_table lock
168  * - wait_event until ref count is zero
169  *
170  * It is the consumer's responsibilty to make sure that no QP
171  * operations (WQE posting or state modification) are pending when a
172  * QP is destroyed. Also, the consumer must make sure that calls to
173  * qp_modify are serialized. Similarly, the consumer is responsible
174  * for ensuring that no CQ resize operations are pending when a CQ
175  * is destroyed.
176  *
177  * Possible optimizations (wait for profile data to see if/where we
178  * have locks bouncing between CPUs):
179  * - split cq/qp table lock into n separate (cache-aligned) locks,
180  * indexed (say) by the page in the table
181  * - split QP struct lock into three (one for common info, one for the
182  * send queue and one for the receive queue)
183  */
184 
185 struct mthca_cq_buf {
187  struct mthca_mr mr;
189 };
190 
193  int cqe;
194  enum {
198  } state;
199 };
200 
201 struct mthca_cq {
202  struct ib_cq ibcq;
204  int refcount;
205  int cqn;
210 
211  /* Next fields are Arbel only */
216  int arm_sn;
217 
219  struct mutex mutex;
220 };
221 
222 struct mthca_srq {
223  struct ib_srq ibsrq;
225  int refcount;
226  int srqn;
227  int max;
228  int max_gs;
232  u16 counter; /* Arbel only */
233  int db_index; /* Arbel only */
234  __be32 *db; /* Arbel only */
235  void *last;
236 
240  struct mthca_mr mr;
241 
243  struct mutex mutex;
244 };
245 
246 struct mthca_wq {
248  int max;
249  unsigned next_ind;
250  unsigned last_comp;
251  unsigned head;
252  unsigned tail;
253  void *last;
254  int max_gs;
256 
257  int db_index; /* Arbel only */
259 };
260 
261 struct mthca_qp {
262  struct ib_qp ibqp;
263  int refcount;
266  u8 port; /* for SQP and memfree use only */
267  u8 alt_port; /* for memfree use only */
272 
273  struct mthca_mr mr;
274 
275  struct mthca_wq rq;
276  struct mthca_wq sq;
280 
283 
285  struct mutex mutex;
286 };
287 
288 struct mthca_sqp {
289  struct mthca_qp qp;
295  void *header_buf;
297 };
298 
299 static inline struct mthca_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
300 {
301  return container_of(ibucontext, struct mthca_ucontext, ibucontext);
302 }
303 
304 static inline struct mthca_fmr *to_mfmr(struct ib_fmr *ibmr)
305 {
306  return container_of(ibmr, struct mthca_fmr, ibmr);
307 }
308 
309 static inline struct mthca_mr *to_mmr(struct ib_mr *ibmr)
310 {
311  return container_of(ibmr, struct mthca_mr, ibmr);
312 }
313 
314 static inline struct mthca_pd *to_mpd(struct ib_pd *ibpd)
315 {
316  return container_of(ibpd, struct mthca_pd, ibpd);
317 }
318 
319 static inline struct mthca_ah *to_mah(struct ib_ah *ibah)
320 {
321  return container_of(ibah, struct mthca_ah, ibah);
322 }
323 
324 static inline struct mthca_cq *to_mcq(struct ib_cq *ibcq)
325 {
326  return container_of(ibcq, struct mthca_cq, ibcq);
327 }
328 
329 static inline struct mthca_srq *to_msrq(struct ib_srq *ibsrq)
330 {
331  return container_of(ibsrq, struct mthca_srq, ibsrq);
332 }
333 
334 static inline struct mthca_qp *to_mqp(struct ib_qp *ibqp)
335 {
336  return container_of(ibqp, struct mthca_qp, ibqp);
337 }
338 
339 static inline struct mthca_sqp *to_msqp(struct mthca_qp *qp)
340 {
341  return container_of(qp, struct mthca_sqp, qp);
342 }
343 
344 #endif /* MTHCA_PROVIDER_H */