Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
qib_sdma.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2012 Intel Corporation. All rights reserved.
3  * Copyright (c) 2007 - 2012 QLogic Corporation. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses. You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  * Redistribution and use in source and binary forms, with or
12  * without modification, are permitted provided that the following
13  * conditions are met:
14  *
15  * - Redistributions of source code must retain the above
16  * copyright notice, this list of conditions and the following
17  * disclaimer.
18  *
19  * - Redistributions in binary form must reproduce the above
20  * copyright notice, this list of conditions and the following
21  * disclaimer in the documentation and/or other materials
22  * provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/spinlock.h>
35 #include <linux/netdevice.h>
36 #include <linux/moduleparam.h>
37 
38 #include "qib.h"
39 #include "qib_common.h"
40 
41 /* default pio off, sdma on */
42 static ushort sdma_descq_cnt = 256;
43 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO);
44 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
45 
46 /*
47  * Bits defined in the send DMA descriptor.
48  */
49 #define SDMA_DESC_LAST (1ULL << 11)
50 #define SDMA_DESC_FIRST (1ULL << 12)
51 #define SDMA_DESC_DMA_HEAD (1ULL << 13)
52 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14)
53 #define SDMA_DESC_INTR (1ULL << 15)
54 #define SDMA_DESC_COUNT_LSB 16
55 #define SDMA_DESC_GEN_LSB 30
56 
58  [qib_sdma_state_s00_hw_down] = "s00_HwDown",
59  [qib_sdma_state_s10_hw_start_up_wait] = "s10_HwStartUpWait",
60  [qib_sdma_state_s20_idle] = "s20_Idle",
61  [qib_sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait",
62  [qib_sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait",
63  [qib_sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait",
64  [qib_sdma_state_s99_running] = "s99_Running",
65 };
66 
68  [qib_sdma_event_e00_go_hw_down] = "e00_GoHwDown",
69  [qib_sdma_event_e10_go_hw_start] = "e10_GoHwStart",
70  [qib_sdma_event_e20_hw_started] = "e20_HwStarted",
71  [qib_sdma_event_e30_go_running] = "e30_GoRunning",
72  [qib_sdma_event_e40_sw_cleaned] = "e40_SwCleaned",
73  [qib_sdma_event_e50_hw_cleaned] = "e50_HwCleaned",
74  [qib_sdma_event_e60_hw_halted] = "e60_HwHalted",
75  [qib_sdma_event_e70_go_idle] = "e70_GoIdle",
76  [qib_sdma_event_e7220_err_halted] = "e7220_ErrHalted",
77  [qib_sdma_event_e7322_err_halted] = "e7322_ErrHalted",
78  [qib_sdma_event_e90_timer_tick] = "e90_TimerTick",
79 };
80 
81 /* declare all statics here rather than keep sorting */
82 static int alloc_sdma(struct qib_pportdata *);
83 static void sdma_complete(struct kref *);
84 static void sdma_finalput(struct qib_sdma_state *);
85 static void sdma_get(struct qib_sdma_state *);
86 static void sdma_put(struct qib_sdma_state *);
87 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states);
88 static void sdma_start_sw_clean_up(struct qib_pportdata *);
89 static void sdma_sw_clean_up_task(unsigned long);
90 static void unmap_desc(struct qib_pportdata *, unsigned);
91 
92 static void sdma_get(struct qib_sdma_state *ss)
93 {
94  kref_get(&ss->kref);
95 }
96 
97 static void sdma_complete(struct kref *kref)
98 {
99  struct qib_sdma_state *ss =
100  container_of(kref, struct qib_sdma_state, kref);
101 
102  complete(&ss->comp);
103 }
104 
105 static void sdma_put(struct qib_sdma_state *ss)
106 {
107  kref_put(&ss->kref, sdma_complete);
108 }
109 
110 static void sdma_finalput(struct qib_sdma_state *ss)
111 {
112  sdma_put(ss);
114 }
115 
116 /*
117  * Complete all the sdma requests on the active list, in the correct
118  * order, and with appropriate processing. Called when cleaning up
119  * after sdma shutdown, and when new sdma requests are submitted for
120  * a link that is down. This matches what is done for requests
121  * that complete normally, it's just the full list.
122  *
123  * Must be called with sdma_lock held
124  */
125 static void clear_sdma_activelist(struct qib_pportdata *ppd)
126 {
127  struct qib_sdma_txreq *txp, *txp_next;
128 
129  list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) {
130  list_del_init(&txp->list);
131  if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) {
132  unsigned idx;
133 
134  idx = txp->start_idx;
135  while (idx != txp->next_descq_idx) {
136  unmap_desc(ppd, idx);
137  if (++idx == ppd->sdma_descq_cnt)
138  idx = 0;
139  }
140  }
141  if (txp->callback)
142  (*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED);
143  }
144 }
145 
146 static void sdma_sw_clean_up_task(unsigned long opaque)
147 {
148  struct qib_pportdata *ppd = (struct qib_pportdata *) opaque;
149  unsigned long flags;
150 
151  spin_lock_irqsave(&ppd->sdma_lock, flags);
152 
153  /*
154  * At this point, the following should always be true:
155  * - We are halted, so no more descriptors are getting retired.
156  * - We are not running, so no one is submitting new work.
157  * - Only we can send the e40_sw_cleaned, so we can't start
158  * running again until we say so. So, the active list and
159  * descq are ours to play with.
160  */
161 
162  /* Process all retired requests. */
164 
165  clear_sdma_activelist(ppd);
166 
167  /*
168  * Resync count of added and removed. It is VERY important that
169  * sdma_descq_removed NEVER decrement - user_sdma depends on it.
170  */
172 
173  /*
174  * Reset our notion of head and tail.
175  * Note that the HW registers will be reset when switching states
176  * due to calling __qib_sdma_process_event() below.
177  */
178  ppd->sdma_descq_tail = 0;
179  ppd->sdma_descq_head = 0;
180  ppd->sdma_head_dma[0] = 0;
181  ppd->sdma_generation = 0;
182 
184 
185  spin_unlock_irqrestore(&ppd->sdma_lock, flags);
186 }
187 
188 /*
189  * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait
190  * as a result of send buffer errors or send DMA descriptor errors.
191  * We want to disarm the buffers in these cases.
192  */
193 static void sdma_hw_start_up(struct qib_pportdata *ppd)
194 {
195  struct qib_sdma_state *ss = &ppd->sdma_state;
196  unsigned bufno;
197 
198  for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno)
199  ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno));
200 
201  ppd->dd->f_sdma_hw_start_up(ppd);
202 }
203 
204 static void sdma_sw_tear_down(struct qib_pportdata *ppd)
205 {
206  struct qib_sdma_state *ss = &ppd->sdma_state;
207 
208  /* Releasing this reference means the state machine has stopped. */
209  sdma_put(ss);
210 }
211 
212 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd)
213 {
214  tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task);
215 }
216 
217 static void sdma_set_state(struct qib_pportdata *ppd,
219 {
220  struct qib_sdma_state *ss = &ppd->sdma_state;
222  unsigned op = 0;
223 
224  /* debugging bookkeeping */
225  ss->previous_state = ss->current_state;
226  ss->previous_op = ss->current_op;
227 
229 
230  if (action[next_state].op_enable)
232 
233  if (action[next_state].op_intenable)
235 
236  if (action[next_state].op_halt)
238 
239  if (action[next_state].op_drain)
241 
242  if (action[next_state].go_s99_running_tofalse)
243  ss->go_s99_running = 0;
244 
245  if (action[next_state].go_s99_running_totrue)
246  ss->go_s99_running = 1;
247 
248  ss->current_op = op;
249 
250  ppd->dd->f_sdma_sendctrl(ppd, ss->current_op);
251 }
252 
253 static void unmap_desc(struct qib_pportdata *ppd, unsigned head)
254 {
255  __le64 *descqp = &ppd->sdma_descq[head].qw[0];
256  u64 desc[2];
258  size_t len;
259 
260  desc[0] = le64_to_cpu(descqp[0]);
261  desc[1] = le64_to_cpu(descqp[1]);
262 
263  addr = (desc[1] << 32) | (desc[0] >> 32);
264  len = (desc[0] >> 14) & (0x7ffULL << 2);
265  dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
266 }
267 
268 static int alloc_sdma(struct qib_pportdata *ppd)
269 {
270  ppd->sdma_descq_cnt = sdma_descq_cnt;
271  if (!ppd->sdma_descq_cnt)
272  ppd->sdma_descq_cnt = 256;
273 
274  /* Allocate memory for SendDMA descriptor FIFO */
275  ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev,
276  ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys,
277  GFP_KERNEL);
278 
279  if (!ppd->sdma_descq) {
280  qib_dev_err(ppd->dd,
281  "failed to allocate SendDMA descriptor FIFO memory\n");
282  goto bail;
283  }
284 
285  /* Allocate memory for DMA of head register to memory */
286  ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev,
288  if (!ppd->sdma_head_dma) {
289  qib_dev_err(ppd->dd,
290  "failed to allocate SendDMA head memory\n");
291  goto cleanup_descq;
292  }
293  ppd->sdma_head_dma[0] = 0;
294  return 0;
295 
296 cleanup_descq:
297  dma_free_coherent(&ppd->dd->pcidev->dev,
298  ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq,
299  ppd->sdma_descq_phys);
300  ppd->sdma_descq = NULL;
301  ppd->sdma_descq_phys = 0;
302 bail:
303  ppd->sdma_descq_cnt = 0;
304  return -ENOMEM;
305 }
306 
307 static void free_sdma(struct qib_pportdata *ppd)
308 {
309  struct qib_devdata *dd = ppd->dd;
310 
311  if (ppd->sdma_head_dma) {
313  (void *)ppd->sdma_head_dma,
314  ppd->sdma_head_phys);
315  ppd->sdma_head_dma = NULL;
316  ppd->sdma_head_phys = 0;
317  }
318 
319  if (ppd->sdma_descq) {
320  dma_free_coherent(&dd->pcidev->dev,
321  ppd->sdma_descq_cnt * sizeof(u64[2]),
322  ppd->sdma_descq, ppd->sdma_descq_phys);
323  ppd->sdma_descq = NULL;
324  ppd->sdma_descq_phys = 0;
325  }
326 }
327 
328 static inline void make_sdma_desc(struct qib_pportdata *ppd,
329  u64 *sdmadesc, u64 addr, u64 dwlen,
330  u64 dwoffset)
331 {
332 
333  WARN_ON(addr & 3);
334  /* SDmaPhyAddr[47:32] */
335  sdmadesc[1] = addr >> 32;
336  /* SDmaPhyAddr[31:0] */
337  sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
338  /* SDmaGeneration[1:0] */
339  sdmadesc[0] |= (ppd->sdma_generation & 3ULL) <<
341  /* SDmaDwordCount[10:0] */
342  sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB;
343  /* SDmaBufOffset[12:2] */
344  sdmadesc[0] |= dwoffset & 0x7ffULL;
345 }
346 
347 /* sdma_lock must be held */
349 {
350  struct list_head *lp = NULL;
351  struct qib_sdma_txreq *txp = NULL;
352  struct qib_devdata *dd = ppd->dd;
353  int progress = 0;
354  u16 hwhead;
355  u16 idx = 0;
356 
357  hwhead = dd->f_sdma_gethead(ppd);
358 
359  /* The reason for some of the complexity of this code is that
360  * not all descriptors have corresponding txps. So, we have to
361  * be able to skip over descs until we wander into the range of
362  * the next txp on the list.
363  */
364 
365  if (!list_empty(&ppd->sdma_activelist)) {
366  lp = ppd->sdma_activelist.next;
367  txp = list_entry(lp, struct qib_sdma_txreq, list);
368  idx = txp->start_idx;
369  }
370 
371  while (ppd->sdma_descq_head != hwhead) {
372  /* if desc is part of this txp, unmap if needed */
373  if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) &&
374  (idx == ppd->sdma_descq_head)) {
375  unmap_desc(ppd, ppd->sdma_descq_head);
376  if (++idx == ppd->sdma_descq_cnt)
377  idx = 0;
378  }
379 
380  /* increment dequed desc count */
381  ppd->sdma_descq_removed++;
382 
383  /* advance head, wrap if needed */
384  if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt)
385  ppd->sdma_descq_head = 0;
386 
387  /* if now past this txp's descs, do the callback */
388  if (txp && txp->next_descq_idx == ppd->sdma_descq_head) {
389  /* remove from active list */
390  list_del_init(&txp->list);
391  if (txp->callback)
392  (*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK);
393  /* see if there is another txp */
394  if (list_empty(&ppd->sdma_activelist))
395  txp = NULL;
396  else {
397  lp = ppd->sdma_activelist.next;
398  txp = list_entry(lp, struct qib_sdma_txreq,
399  list);
400  idx = txp->start_idx;
401  }
402  }
403  progress = 1;
404  }
405  if (progress)
406  qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
407  return progress;
408 }
409 
410 /*
411  * This is called from interrupt context.
412  */
413 void qib_sdma_intr(struct qib_pportdata *ppd)
414 {
415  unsigned long flags;
416 
417  spin_lock_irqsave(&ppd->sdma_lock, flags);
418 
419  __qib_sdma_intr(ppd);
420 
421  spin_unlock_irqrestore(&ppd->sdma_lock, flags);
422 }
423 
425 {
426  if (__qib_sdma_running(ppd))
428 }
429 
431 {
432  struct qib_devdata *dd = ppd->dd;
433  unsigned long flags;
434  int ret = 0;
435 
436  ret = alloc_sdma(ppd);
437  if (ret)
438  goto bail;
439 
440  /* set consistent sdma state */
441  ppd->dd->f_sdma_init_early(ppd);
442  spin_lock_irqsave(&ppd->sdma_lock, flags);
443  sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
444  spin_unlock_irqrestore(&ppd->sdma_lock, flags);
445 
446  /* set up reference counting */
447  kref_init(&ppd->sdma_state.kref);
448  init_completion(&ppd->sdma_state.comp);
449 
450  ppd->sdma_generation = 0;
451  ppd->sdma_descq_head = 0;
452  ppd->sdma_descq_removed = 0;
453  ppd->sdma_descq_added = 0;
454 
455  INIT_LIST_HEAD(&ppd->sdma_activelist);
456 
457  tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
458  (unsigned long)ppd);
459 
460  ret = dd->f_init_sdma_regs(ppd);
461  if (ret)
462  goto bail_alloc;
463 
465 
466  return 0;
467 
468 bail_alloc:
469  qib_teardown_sdma(ppd);
470 bail:
471  return ret;
472 }
473 
475 {
477 
478  /*
479  * This waits for the state machine to exit so it is not
480  * necessary to kill the sdma_sw_clean_up_task to make sure
481  * it is not running.
482  */
483  sdma_finalput(&ppd->sdma_state);
484 
485  free_sdma(ppd);
486 }
487 
489 {
490  unsigned long flags;
491  int ret;
492 
493  spin_lock_irqsave(&ppd->sdma_lock, flags);
494  ret = __qib_sdma_running(ppd);
495  spin_unlock_irqrestore(&ppd->sdma_lock, flags);
496 
497  return ret;
498 }
499 
500 /*
501  * Complete a request when sdma not running; likely only request
502  * but to simplify the code, always queue it, then process the full
503  * activelist. We process the entire list to ensure that this particular
504  * request does get it's callback, but in the correct order.
505  * Must be called with sdma_lock held
506  */
507 static void complete_sdma_err_req(struct qib_pportdata *ppd,
508  struct qib_verbs_txreq *tx)
509 {
510  atomic_inc(&tx->qp->s_dma_busy);
511  /* no sdma descriptors, so no unmap_desc */
512  tx->txreq.start_idx = 0;
513  tx->txreq.next_descq_idx = 0;
514  list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
515  clear_sdma_activelist(ppd);
516 }
517 
518 /*
519  * This function queues one IB packet onto the send DMA queue per call.
520  * The caller is responsible for checking:
521  * 1) The number of send DMA descriptor entries is less than the size of
522  * the descriptor queue.
523  * 2) The IB SGE addresses and lengths are 32-bit aligned
524  * (except possibly the last SGE's length)
525  * 3) The SGE addresses are suitable for passing to dma_map_single().
526  */
528  struct qib_sge_state *ss, u32 dwords,
529  struct qib_verbs_txreq *tx)
530 {
531  unsigned long flags;
532  struct qib_sge *sge;
533  struct qib_qp *qp;
534  int ret = 0;
535  u16 tail;
536  __le64 *descqp;
537  u64 sdmadesc[2];
538  u32 dwoffset;
540 
541  spin_lock_irqsave(&ppd->sdma_lock, flags);
542 
543 retry:
544  if (unlikely(!__qib_sdma_running(ppd))) {
545  complete_sdma_err_req(ppd, tx);
546  goto unlock;
547  }
548 
549  if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) {
550  if (qib_sdma_make_progress(ppd))
551  goto retry;
552  if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT)
553  ppd->dd->f_sdma_set_desc_cnt(ppd,
554  ppd->sdma_descq_cnt / 2);
555  goto busy;
556  }
557 
558  dwoffset = tx->hdr_dwords;
559  make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0);
560 
561  sdmadesc[0] |= SDMA_DESC_FIRST;
562  if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
563  sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
564 
565  /* write to the descq */
566  tail = ppd->sdma_descq_tail;
567  descqp = &ppd->sdma_descq[tail].qw[0];
568  *descqp++ = cpu_to_le64(sdmadesc[0]);
569  *descqp++ = cpu_to_le64(sdmadesc[1]);
570 
571  /* increment the tail */
572  if (++tail == ppd->sdma_descq_cnt) {
573  tail = 0;
574  descqp = &ppd->sdma_descq[0].qw[0];
575  ++ppd->sdma_generation;
576  }
577 
578  tx->txreq.start_idx = tail;
579 
580  sge = &ss->sge;
581  while (dwords) {
582  u32 dw;
583  u32 len;
584 
585  len = dwords << 2;
586  if (len > sge->length)
587  len = sge->length;
588  if (len > sge->sge_length)
589  len = sge->sge_length;
590  BUG_ON(len == 0);
591  dw = (len + 3) >> 2;
592  addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr,
593  dw << 2, DMA_TO_DEVICE);
594  if (dma_mapping_error(&ppd->dd->pcidev->dev, addr))
595  goto unmap;
596  sdmadesc[0] = 0;
597  make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset);
598  /* SDmaUseLargeBuf has to be set in every descriptor */
599  if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
600  sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
601  /* write to the descq */
602  *descqp++ = cpu_to_le64(sdmadesc[0]);
603  *descqp++ = cpu_to_le64(sdmadesc[1]);
604 
605  /* increment the tail */
606  if (++tail == ppd->sdma_descq_cnt) {
607  tail = 0;
608  descqp = &ppd->sdma_descq[0].qw[0];
609  ++ppd->sdma_generation;
610  }
611  sge->vaddr += len;
612  sge->length -= len;
613  sge->sge_length -= len;
614  if (sge->sge_length == 0) {
615  if (--ss->num_sge)
616  *sge = *ss->sg_list++;
617  } else if (sge->length == 0 && sge->mr->lkey) {
618  if (++sge->n >= QIB_SEGSZ) {
619  if (++sge->m >= sge->mr->mapsz)
620  break;
621  sge->n = 0;
622  }
623  sge->vaddr =
624  sge->mr->map[sge->m]->segs[sge->n].vaddr;
625  sge->length =
626  sge->mr->map[sge->m]->segs[sge->n].length;
627  }
628 
629  dwoffset += dw;
630  dwords -= dw;
631  }
632 
633  if (!tail)
634  descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0];
635  descqp -= 2;
636  descqp[0] |= cpu_to_le64(SDMA_DESC_LAST);
637  if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST)
638  descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD);
639  if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ)
640  descqp[0] |= cpu_to_le64(SDMA_DESC_INTR);
641 
642  atomic_inc(&tx->qp->s_dma_busy);
643  tx->txreq.next_descq_idx = tail;
644  ppd->dd->f_sdma_update_tail(ppd, tail);
645  ppd->sdma_descq_added += tx->txreq.sg_count;
646  list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
647  goto unlock;
648 
649 unmap:
650  for (;;) {
651  if (!tail)
652  tail = ppd->sdma_descq_cnt - 1;
653  else
654  tail--;
655  if (tail == ppd->sdma_descq_tail)
656  break;
657  unmap_desc(ppd, tail);
658  }
659  qp = tx->qp;
660  qib_put_txreq(tx);
661  spin_lock(&qp->r_lock);
662  spin_lock(&qp->s_lock);
663  if (qp->ibqp.qp_type == IB_QPT_RC) {
664  /* XXX what about error sending RDMA read responses? */
667  } else if (qp->s_wqe)
669  spin_unlock(&qp->s_lock);
670  spin_unlock(&qp->r_lock);
671  /* return zero to process the next send work request */
672  goto unlock;
673 
674 busy:
675  qp = tx->qp;
676  spin_lock(&qp->s_lock);
678  struct qib_ibdev *dev;
679 
680  /*
681  * If we couldn't queue the DMA request, save the info
682  * and try again later rather than destroying the
683  * buffer and undoing the side effects of the copy.
684  */
685  tx->ss = ss;
686  tx->dwords = dwords;
687  qp->s_tx = tx;
688  dev = &ppd->dd->verbs_dev;
689  spin_lock(&dev->pending_lock);
690  if (list_empty(&qp->iowait)) {
691  struct qib_ibport *ibp;
692 
693  ibp = &ppd->ibport_data;
694  ibp->n_dmawait++;
696  list_add_tail(&qp->iowait, &dev->dmawait);
697  }
698  spin_unlock(&dev->pending_lock);
699  qp->s_flags &= ~QIB_S_BUSY;
700  spin_unlock(&qp->s_lock);
701  ret = -EBUSY;
702  } else {
703  spin_unlock(&qp->s_lock);
704  qib_put_txreq(tx);
705  }
706 unlock:
707  spin_unlock_irqrestore(&ppd->sdma_lock, flags);
708  return ret;
709 }
710 
712  enum qib_sdma_events event)
713 {
714  unsigned long flags;
715 
716  spin_lock_irqsave(&ppd->sdma_lock, flags);
717 
718  __qib_sdma_process_event(ppd, event);
719 
720  if (ppd->sdma_state.current_state == qib_sdma_state_s99_running)
721  qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
722 
723  spin_unlock_irqrestore(&ppd->sdma_lock, flags);
724 }
725 
727  enum qib_sdma_events event)
728 {
729  struct qib_sdma_state *ss = &ppd->sdma_state;
730 
731  switch (ss->current_state) {
733  switch (event) {
735  break;
737  /*
738  * If down, but running requested (usually result
739  * of link up, then we need to start up.
740  * This can happen when hw down is requested while
741  * bringing the link up with traffic active on
742  * 7220, e.g. */
743  ss->go_s99_running = 1;
744  /* fall through and start dma engine */
746  /* This reference means the state machine is started */
747  sdma_get(&ppd->sdma_state);
748  sdma_set_state(ppd,
750  break;
752  break;
754  sdma_sw_tear_down(ppd);
755  break;
757  break;
759  break;
761  break;
763  break;
765  break;
767  break;
768  }
769  break;
770 
772  switch (event) {
774  sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
775  sdma_sw_tear_down(ppd);
776  break;
778  break;
780  sdma_set_state(ppd, ss->go_s99_running ?
783  break;
785  ss->go_s99_running = 1;
786  break;
788  break;
790  break;
792  break;
794  ss->go_s99_running = 0;
795  break;
797  break;
799  break;
801  break;
802  }
803  break;
804 
806  switch (event) {
808  sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
809  sdma_sw_tear_down(ppd);
810  break;
812  break;
814  break;
816  sdma_set_state(ppd, qib_sdma_state_s99_running);
817  ss->go_s99_running = 1;
818  break;
820  break;
822  break;
824  break;
826  break;
828  break;
830  break;
832  break;
833  }
834  break;
835 
837  switch (event) {
839  sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
840  break;
842  break;
844  break;
846  ss->go_s99_running = 1;
847  break;
849  sdma_set_state(ppd,
851  sdma_hw_start_up(ppd);
852  break;
854  break;
856  break;
858  ss->go_s99_running = 0;
859  break;
861  break;
863  break;
865  break;
866  }
867  break;
868 
870  switch (event) {
872  sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
873  sdma_start_sw_clean_up(ppd);
874  break;
876  break;
878  break;
880  ss->go_s99_running = 1;
881  break;
883  break;
885  sdma_set_state(ppd,
887  sdma_start_sw_clean_up(ppd);
888  break;
890  break;
892  ss->go_s99_running = 0;
893  break;
895  break;
897  break;
899  break;
900  }
901  break;
902 
904  switch (event) {
906  sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
907  sdma_start_sw_clean_up(ppd);
908  break;
910  break;
912  break;
914  ss->go_s99_running = 1;
915  break;
917  break;
919  break;
921  sdma_set_state(ppd,
923  ppd->dd->f_sdma_hw_clean_up(ppd);
924  break;
926  ss->go_s99_running = 0;
927  break;
929  break;
931  break;
933  break;
934  }
935  break;
936 
938  switch (event) {
940  sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
941  sdma_start_sw_clean_up(ppd);
942  break;
944  break;
946  break;
948  break;
950  break;
952  break;
954  sdma_set_state(ppd,
956  sdma_start_sw_clean_up(ppd);
957  break;
959  sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
960  ss->go_s99_running = 0;
961  break;
963  sdma_set_state(ppd,
965  sdma_start_sw_clean_up(ppd);
966  break;
968  sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
969  break;
971  break;
972  }
973  break;
974  }
975 
976  ss->last_event = event;
977 }