Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
dma.c
Go to the documentation of this file.
1 /*
2  * Intel I/OAT DMA Linux driver
3  * Copyright(c) 2004 - 2009 Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17  *
18  * The full GNU General Public License is included in this distribution in
19  * the file called "COPYING".
20  *
21  */
22 
23 /*
24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
25  * copy operations.
26  */
27 
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/slab.h>
31 #include <linux/pci.h>
32 #include <linux/interrupt.h>
33 #include <linux/dmaengine.h>
34 #include <linux/delay.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/workqueue.h>
37 #include <linux/prefetch.h>
38 #include <linux/i7300_idle.h>
39 #include "dma.h"
40 #include "registers.h"
41 #include "hw.h"
42 
43 #include "../dmaengine.h"
44 
48  "high-water mark for pushing ioat descriptors (default: 4)");
49 
50 /* internal functions */
51 static void ioat1_cleanup(struct ioat_dma_chan *ioat);
52 static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
53 
59 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
60 {
61  struct ioatdma_device *instance = data;
62  struct ioat_chan_common *chan;
63  unsigned long attnstatus;
64  int bit;
65  u8 intrctrl;
66 
67  intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
68 
69  if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
70  return IRQ_NONE;
71 
72  if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
73  writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
74  return IRQ_NONE;
75  }
76 
77  attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
78  for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
79  chan = ioat_chan_by_index(instance, bit);
80  tasklet_schedule(&chan->cleanup_task);
81  }
82 
83  writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
84  return IRQ_HANDLED;
85 }
86 
92 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
93 {
94  struct ioat_chan_common *chan = data;
95 
96  tasklet_schedule(&chan->cleanup_task);
97 
98  return IRQ_HANDLED;
99 }
100 
101 /* common channel initialization */
103 {
104  struct dma_device *dma = &device->common;
105  struct dma_chan *c = &chan->common;
106  unsigned long data = (unsigned long) c;
107 
108  chan->device = device;
109  chan->reg_base = device->reg_base + (0x80 * (idx + 1));
111  chan->common.device = dma;
112  dma_cookie_init(&chan->common);
113  list_add_tail(&chan->common.device_node, &dma->channels);
114  device->idx[idx] = chan;
115  init_timer(&chan->timer);
116  chan->timer.function = device->timer_fn;
117  chan->timer.data = data;
118  tasklet_init(&chan->cleanup_task, device->cleanup_fn, data);
119  tasklet_disable(&chan->cleanup_task);
120 }
121 
126 static int ioat1_enumerate_channels(struct ioatdma_device *device)
127 {
128  u8 xfercap_scale;
129  u32 xfercap;
130  int i;
131  struct ioat_dma_chan *ioat;
132  struct device *dev = &device->pdev->dev;
133  struct dma_device *dma = &device->common;
134 
135  INIT_LIST_HEAD(&dma->channels);
136  dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
137  dma->chancnt &= 0x1f; /* bits [4:0] valid */
138  if (dma->chancnt > ARRAY_SIZE(device->idx)) {
139  dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
140  dma->chancnt, ARRAY_SIZE(device->idx));
141  dma->chancnt = ARRAY_SIZE(device->idx);
142  }
143  xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
144  xfercap_scale &= 0x1f; /* bits [4:0] valid */
145  xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
146  dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
147 
148 #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
149  if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
150  dma->chancnt--;
151 #endif
152  for (i = 0; i < dma->chancnt; i++) {
153  ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
154  if (!ioat)
155  break;
156 
157  ioat_init_channel(device, &ioat->base, i);
158  ioat->xfercap = xfercap;
159  spin_lock_init(&ioat->desc_lock);
160  INIT_LIST_HEAD(&ioat->free_desc);
161  INIT_LIST_HEAD(&ioat->used_desc);
162  }
163  dma->chancnt = i;
164  return i;
165 }
166 
172 static inline void
173 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
174 {
175  void __iomem *reg_base = ioat->base.reg_base;
176 
177  dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
178  __func__, ioat->pending);
179  ioat->pending = 0;
181 }
182 
183 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
184 {
185  struct ioat_dma_chan *ioat = to_ioat_chan(chan);
186 
187  if (ioat->pending > 0) {
188  spin_lock_bh(&ioat->desc_lock);
189  __ioat1_dma_memcpy_issue_pending(ioat);
190  spin_unlock_bh(&ioat->desc_lock);
191  }
192 }
193 
198 static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
199 {
200  struct ioat_chan_common *chan = &ioat->base;
201  void __iomem *reg_base = chan->reg_base;
202  u32 chansts, chanerr;
203 
204  dev_warn(to_dev(chan), "reset\n");
205  chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
206  chansts = *chan->completion & IOAT_CHANSTS_STATUS;
207  if (chanerr) {
208  dev_err(to_dev(chan),
209  "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
210  chan_num(chan), chansts, chanerr);
211  writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
212  }
213 
214  /*
215  * whack it upside the head with a reset
216  * and wait for things to settle out.
217  * force the pending count to a really big negative
218  * to make sure no one forces an issue_pending
219  * while we're waiting.
220  */
221 
222  ioat->pending = INT_MIN;
224  reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
226  mod_timer(&chan->timer, jiffies + RESET_DELAY);
227 }
228 
229 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
230 {
231  struct dma_chan *c = tx->chan;
232  struct ioat_dma_chan *ioat = to_ioat_chan(c);
233  struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
234  struct ioat_chan_common *chan = &ioat->base;
235  struct ioat_desc_sw *first;
236  struct ioat_desc_sw *chain_tail;
238 
239  spin_lock_bh(&ioat->desc_lock);
240  /* cookie incr and addition to used_list must be atomic */
241  cookie = dma_cookie_assign(tx);
242  dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
243 
244  /* write address into NextDescriptor field of last desc in chain */
245  first = to_ioat_desc(desc->tx_list.next);
246  chain_tail = to_ioat_desc(ioat->used_desc.prev);
247  /* make descriptor updates globally visible before chaining */
248  wmb();
249  chain_tail->hw->next = first->txd.phys;
250  list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
251  dump_desc_dbg(ioat, chain_tail);
252  dump_desc_dbg(ioat, first);
253 
255  mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
256 
257  ioat->active += desc->hw->tx_cnt;
258  ioat->pending += desc->hw->tx_cnt;
259  if (ioat->pending >= ioat_pending_level)
260  __ioat1_dma_memcpy_issue_pending(ioat);
261  spin_unlock_bh(&ioat->desc_lock);
262 
263  return cookie;
264 }
265 
271 static struct ioat_desc_sw *
272 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
273 {
274  struct ioat_dma_descriptor *desc;
275  struct ioat_desc_sw *desc_sw;
278 
279  ioatdma_device = ioat->base.device;
280  desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
281  if (unlikely(!desc))
282  return NULL;
283 
284  desc_sw = kzalloc(sizeof(*desc_sw), flags);
285  if (unlikely(!desc_sw)) {
286  pci_pool_free(ioatdma_device->dma_pool, desc, phys);
287  return NULL;
288  }
289 
290  memset(desc, 0, sizeof(*desc));
291 
292  INIT_LIST_HEAD(&desc_sw->tx_list);
293  dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
294  desc_sw->txd.tx_submit = ioat1_tx_submit;
295  desc_sw->hw = desc;
296  desc_sw->txd.phys = phys;
297  set_desc_id(desc_sw, -1);
298 
299  return desc_sw;
300 }
301 
302 static int ioat_initial_desc_count = 256;
303 module_param(ioat_initial_desc_count, int, 0644);
304 MODULE_PARM_DESC(ioat_initial_desc_count,
305  "ioat1: initial descriptors per channel (default: 256)");
310 static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
311 {
312  struct ioat_dma_chan *ioat = to_ioat_chan(c);
313  struct ioat_chan_common *chan = &ioat->base;
314  struct ioat_desc_sw *desc;
315  u32 chanerr;
316  int i;
317  LIST_HEAD(tmp_list);
318 
319  /* have we already been set up? */
320  if (!list_empty(&ioat->free_desc))
321  return ioat->desccount;
322 
323  /* Setup register to interrupt and write completion status on error */
325 
326  chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
327  if (chanerr) {
328  dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
329  writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
330  }
331 
332  /* Allocate descriptors */
333  for (i = 0; i < ioat_initial_desc_count; i++) {
334  desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
335  if (!desc) {
336  dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
337  break;
338  }
339  set_desc_id(desc, i);
340  list_add_tail(&desc->node, &tmp_list);
341  }
342  spin_lock_bh(&ioat->desc_lock);
343  ioat->desccount = i;
344  list_splice(&tmp_list, &ioat->free_desc);
345  spin_unlock_bh(&ioat->desc_lock);
346 
347  /* allocate a completion writeback area */
348  /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
349  chan->completion = pci_pool_alloc(chan->device->completion_pool,
350  GFP_KERNEL, &chan->completion_dma);
351  memset(chan->completion, 0, sizeof(*chan->completion));
352  writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
354  writel(((u64) chan->completion_dma) >> 32,
356 
357  tasklet_enable(&chan->cleanup_task);
358  ioat1_dma_start_null_desc(ioat); /* give chain to dma device */
359  dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
360  __func__, ioat->desccount);
361  return ioat->desccount;
362 }
363 
368 static void ioat1_dma_free_chan_resources(struct dma_chan *c)
369 {
370  struct ioat_dma_chan *ioat = to_ioat_chan(c);
371  struct ioat_chan_common *chan = &ioat->base;
372  struct ioatdma_device *ioatdma_device = chan->device;
373  struct ioat_desc_sw *desc, *_desc;
374  int in_use_descs = 0;
375 
376  /* Before freeing channel resources first check
377  * if they have been previously allocated for this channel.
378  */
379  if (ioat->desccount == 0)
380  return;
381 
382  tasklet_disable(&chan->cleanup_task);
383  del_timer_sync(&chan->timer);
384  ioat1_cleanup(ioat);
385 
386  /* Delay 100ms after reset to allow internal DMA logic to quiesce
387  * before removing DMA descriptor resources.
388  */
390  chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
391  mdelay(100);
392 
393  spin_lock_bh(&ioat->desc_lock);
394  list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
395  dev_dbg(to_dev(chan), "%s: freeing %d from used list\n",
396  __func__, desc_id(desc));
397  dump_desc_dbg(ioat, desc);
398  in_use_descs++;
399  list_del(&desc->node);
400  pci_pool_free(ioatdma_device->dma_pool, desc->hw,
401  desc->txd.phys);
402  kfree(desc);
403  }
404  list_for_each_entry_safe(desc, _desc,
405  &ioat->free_desc, node) {
406  list_del(&desc->node);
407  pci_pool_free(ioatdma_device->dma_pool, desc->hw,
408  desc->txd.phys);
409  kfree(desc);
410  }
411  spin_unlock_bh(&ioat->desc_lock);
412 
413  pci_pool_free(ioatdma_device->completion_pool,
414  chan->completion,
415  chan->completion_dma);
416 
417  /* one is ok since we left it on there on purpose */
418  if (in_use_descs > 1)
419  dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
420  in_use_descs - 1);
421 
422  chan->last_completion = 0;
423  chan->completion_dma = 0;
424  ioat->pending = 0;
425  ioat->desccount = 0;
426 }
427 
436 static struct ioat_desc_sw *
437 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
438 {
439  struct ioat_desc_sw *new;
440 
441  if (!list_empty(&ioat->free_desc)) {
442  new = to_ioat_desc(ioat->free_desc.next);
443  list_del(&new->node);
444  } else {
445  /* try to get another desc */
446  new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
447  if (!new) {
448  dev_err(to_dev(&ioat->base), "alloc failed\n");
449  return NULL;
450  }
451  }
452  dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n",
453  __func__, desc_id(new));
454  prefetch(new->hw);
455  return new;
456 }
457 
458 static struct dma_async_tx_descriptor *
459 ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
460  dma_addr_t dma_src, size_t len, unsigned long flags)
461 {
462  struct ioat_dma_chan *ioat = to_ioat_chan(c);
463  struct ioat_desc_sw *desc;
464  size_t copy;
465  LIST_HEAD(chain);
466  dma_addr_t src = dma_src;
467  dma_addr_t dest = dma_dest;
468  size_t total_len = len;
469  struct ioat_dma_descriptor *hw = NULL;
470  int tx_cnt = 0;
471 
472  spin_lock_bh(&ioat->desc_lock);
473  desc = ioat1_dma_get_next_descriptor(ioat);
474  do {
475  if (!desc)
476  break;
477 
478  tx_cnt++;
479  copy = min_t(size_t, len, ioat->xfercap);
480 
481  hw = desc->hw;
482  hw->size = copy;
483  hw->ctl = 0;
484  hw->src_addr = src;
485  hw->dst_addr = dest;
486 
487  list_add_tail(&desc->node, &chain);
488 
489  len -= copy;
490  dest += copy;
491  src += copy;
492  if (len) {
493  struct ioat_desc_sw *next;
494 
495  async_tx_ack(&desc->txd);
496  next = ioat1_dma_get_next_descriptor(ioat);
497  hw->next = next ? next->txd.phys : 0;
498  dump_desc_dbg(ioat, desc);
499  desc = next;
500  } else
501  hw->next = 0;
502  } while (len);
503 
504  if (!desc) {
505  struct ioat_chan_common *chan = &ioat->base;
506 
507  dev_err(to_dev(chan),
508  "chan%d - get_next_desc failed\n", chan_num(chan));
509  list_splice(&chain, &ioat->free_desc);
510  spin_unlock_bh(&ioat->desc_lock);
511  return NULL;
512  }
513  spin_unlock_bh(&ioat->desc_lock);
514 
515  desc->txd.flags = flags;
516  desc->len = total_len;
517  list_splice(&chain, &desc->tx_list);
518  hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
519  hw->ctl_f.compl_write = 1;
520  hw->tx_cnt = tx_cnt;
521  dump_desc_dbg(ioat, desc);
522 
523  return &desc->txd;
524 }
525 
526 static void ioat1_cleanup_event(unsigned long data)
527 {
528  struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);
529 
530  ioat1_cleanup(ioat);
532 }
533 
534 void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
535  size_t len, struct ioat_dma_descriptor *hw)
536 {
537  struct pci_dev *pdev = chan->device->pdev;
538  size_t offset = len - hw->size;
539 
540  if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
541  ioat_unmap(pdev, hw->dst_addr - offset, len,
542  PCI_DMA_FROMDEVICE, flags, 1);
543 
544  if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
545  ioat_unmap(pdev, hw->src_addr - offset, len,
546  PCI_DMA_TODEVICE, flags, 0);
547 }
548 
550 {
551  dma_addr_t phys_complete;
552  u64 completion;
553 
554  completion = *chan->completion;
555  phys_complete = ioat_chansts_to_addr(completion);
556 
557  dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
558  (unsigned long long) phys_complete);
559 
560  if (is_ioat_halted(completion)) {
561  u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
562  dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
563  chanerr);
564 
565  /* TODO do something to salvage the situation */
566  }
567 
568  return phys_complete;
569 }
570 
572  dma_addr_t *phys_complete)
573 {
574  *phys_complete = ioat_get_current_completion(chan);
575  if (*phys_complete == chan->last_completion)
576  return false;
579 
580  return true;
581 }
582 
583 static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete)
584 {
585  struct ioat_chan_common *chan = &ioat->base;
586  struct list_head *_desc, *n;
587  struct dma_async_tx_descriptor *tx;
588 
589  dev_dbg(to_dev(chan), "%s: phys_complete: %llx\n",
590  __func__, (unsigned long long) phys_complete);
591  list_for_each_safe(_desc, n, &ioat->used_desc) {
592  struct ioat_desc_sw *desc;
593 
594  prefetch(n);
595  desc = list_entry(_desc, typeof(*desc), node);
596  tx = &desc->txd;
597  /*
598  * Incoming DMA requests may use multiple descriptors,
599  * due to exceeding xfercap, perhaps. If so, only the
600  * last one will have a cookie, and require unmapping.
601  */
602  dump_desc_dbg(ioat, desc);
603  if (tx->cookie) {
604  dma_cookie_complete(tx);
605  ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
606  ioat->active -= desc->hw->tx_cnt;
607  if (tx->callback) {
608  tx->callback(tx->callback_param);
609  tx->callback = NULL;
610  }
611  }
612 
613  if (tx->phys != phys_complete) {
614  /*
615  * a completed entry, but not the last, so clean
616  * up if the client is done with the descriptor
617  */
618  if (async_tx_test_ack(tx))
619  list_move_tail(&desc->node, &ioat->free_desc);
620  } else {
621  /*
622  * last used desc. Do not remove, so we can
623  * append from it.
624  */
625 
626  /* if nothing else is pending, cancel the
627  * completion timeout
628  */
629  if (n == &ioat->used_desc) {
630  dev_dbg(to_dev(chan),
631  "%s cancel completion timeout\n",
632  __func__);
634  }
635 
636  /* TODO check status bits? */
637  break;
638  }
639  }
640 
641  chan->last_completion = phys_complete;
642 }
643 
652 static void ioat1_cleanup(struct ioat_dma_chan *ioat)
653 {
654  struct ioat_chan_common *chan = &ioat->base;
655  dma_addr_t phys_complete;
656 
657  prefetch(chan->completion);
658 
659  if (!spin_trylock_bh(&chan->cleanup_lock))
660  return;
661 
662  if (!ioat_cleanup_preamble(chan, &phys_complete)) {
663  spin_unlock_bh(&chan->cleanup_lock);
664  return;
665  }
666 
667  if (!spin_trylock_bh(&ioat->desc_lock)) {
668  spin_unlock_bh(&chan->cleanup_lock);
669  return;
670  }
671 
672  __cleanup(ioat, phys_complete);
673 
674  spin_unlock_bh(&ioat->desc_lock);
675  spin_unlock_bh(&chan->cleanup_lock);
676 }
677 
678 static void ioat1_timer_event(unsigned long data)
679 {
680  struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);
681  struct ioat_chan_common *chan = &ioat->base;
682 
683  dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
684 
685  spin_lock_bh(&chan->cleanup_lock);
687  struct ioat_desc_sw *desc;
688 
689  spin_lock_bh(&ioat->desc_lock);
690 
691  /* restart active descriptors */
692  desc = to_ioat_desc(ioat->used_desc.prev);
693  ioat_set_chainaddr(ioat, desc->txd.phys);
694  ioat_start(chan);
695 
696  ioat->pending = 0;
698  mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
699  spin_unlock_bh(&ioat->desc_lock);
700  } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
701  dma_addr_t phys_complete;
702 
703  spin_lock_bh(&ioat->desc_lock);
704  /* if we haven't made progress and we have already
705  * acknowledged a pending completion once, then be more
706  * forceful with a restart
707  */
708  if (ioat_cleanup_preamble(chan, &phys_complete))
709  __cleanup(ioat, phys_complete);
710  else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
711  ioat1_reset_channel(ioat);
712  else {
713  u64 status = ioat_chansts(chan);
714 
715  /* manually update the last completion address */
716  if (ioat_chansts_to_addr(status) != 0)
717  *chan->completion = status;
718 
720  mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
721  }
722  spin_unlock_bh(&ioat->desc_lock);
723  }
724  spin_unlock_bh(&chan->cleanup_lock);
725 }
726 
727 enum dma_status
729  struct dma_tx_state *txstate)
730 {
731  struct ioat_chan_common *chan = to_chan_common(c);
732  struct ioatdma_device *device = chan->device;
733  enum dma_status ret;
734 
735  ret = dma_cookie_status(c, cookie, txstate);
736  if (ret == DMA_SUCCESS)
737  return ret;
738 
739  device->cleanup_fn((unsigned long) c);
740 
741  return dma_cookie_status(c, cookie, txstate);
742 }
743 
744 static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
745 {
746  struct ioat_chan_common *chan = &ioat->base;
747  struct ioat_desc_sw *desc;
748  struct ioat_dma_descriptor *hw;
749 
750  spin_lock_bh(&ioat->desc_lock);
751 
752  desc = ioat1_dma_get_next_descriptor(ioat);
753 
754  if (!desc) {
755  dev_err(to_dev(chan),
756  "Unable to start null desc - get next desc failed\n");
757  spin_unlock_bh(&ioat->desc_lock);
758  return;
759  }
760 
761  hw = desc->hw;
762  hw->ctl = 0;
763  hw->ctl_f.null = 1;
764  hw->ctl_f.int_en = 1;
765  hw->ctl_f.compl_write = 1;
766  /* set size to non-zero value (channel returns error when size is 0) */
768  hw->src_addr = 0;
769  hw->dst_addr = 0;
770  async_tx_ack(&desc->txd);
771  hw->next = 0;
772  list_add_tail(&desc->node, &ioat->used_desc);
773  dump_desc_dbg(ioat, desc);
774 
775  ioat_set_chainaddr(ioat, desc->txd.phys);
776  ioat_start(chan);
777  spin_unlock_bh(&ioat->desc_lock);
778 }
779 
780 /*
781  * Perform a IOAT transaction to verify the HW works.
782  */
783 #define IOAT_TEST_SIZE 2000
784 
785 static void __devinit ioat_dma_test_callback(void *dma_async_param)
786 {
787  struct completion *cmp = dma_async_param;
788 
789  complete(cmp);
790 }
791 
796 int __devinit ioat_dma_self_test(struct ioatdma_device *device)
797 {
798  int i;
799  u8 *src;
800  u8 *dest;
801  struct dma_device *dma = &device->common;
802  struct device *dev = &device->pdev->dev;
803  struct dma_chan *dma_chan;
804  struct dma_async_tx_descriptor *tx;
805  dma_addr_t dma_dest, dma_src;
807  int err = 0;
808  struct completion cmp;
809  unsigned long tmo;
810  unsigned long flags;
811 
812  src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
813  if (!src)
814  return -ENOMEM;
815  dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
816  if (!dest) {
817  kfree(src);
818  return -ENOMEM;
819  }
820 
821  /* Fill in src buffer */
822  for (i = 0; i < IOAT_TEST_SIZE; i++)
823  src[i] = (u8)i;
824 
825  /* Start copy, using first DMA channel */
826  dma_chan = container_of(dma->channels.next, struct dma_chan,
827  device_node);
828  if (dma->device_alloc_chan_resources(dma_chan) < 1) {
829  dev_err(dev, "selftest cannot allocate chan resource\n");
830  err = -ENODEV;
831  goto out;
832  }
833 
834  dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
835  dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
838  tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
839  IOAT_TEST_SIZE, flags);
840  if (!tx) {
841  dev_err(dev, "Self-test prep failed, disabling\n");
842  err = -ENODEV;
843  goto free_resources;
844  }
845 
846  async_tx_ack(tx);
847  init_completion(&cmp);
848  tx->callback = ioat_dma_test_callback;
849  tx->callback_param = &cmp;
850  cookie = tx->tx_submit(tx);
851  if (cookie < 0) {
852  dev_err(dev, "Self-test setup failed, disabling\n");
853  err = -ENODEV;
854  goto free_resources;
855  }
856  dma->device_issue_pending(dma_chan);
857 
859 
860  if (tmo == 0 ||
861  dma->device_tx_status(dma_chan, cookie, NULL)
862  != DMA_SUCCESS) {
863  dev_err(dev, "Self-test copy timed out, disabling\n");
864  err = -ENODEV;
865  goto free_resources;
866  }
867  if (memcmp(src, dest, IOAT_TEST_SIZE)) {
868  dev_err(dev, "Self-test copy failed compare, disabling\n");
869  err = -ENODEV;
870  goto free_resources;
871  }
872 
873 free_resources:
874  dma->device_free_chan_resources(dma_chan);
875 out:
876  kfree(src);
877  kfree(dest);
878  return err;
879 }
880 
881 static char ioat_interrupt_style[32] = "msix";
882 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
883  sizeof(ioat_interrupt_style), 0644);
884 MODULE_PARM_DESC(ioat_interrupt_style,
885  "set ioat interrupt style: msix (default), "
886  "msix-single-vector, msi, intx)");
887 
892 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
893 {
894  struct ioat_chan_common *chan;
895  struct pci_dev *pdev = device->pdev;
896  struct device *dev = &pdev->dev;
897  struct msix_entry *msix;
898  int i, j, msixcnt;
899  int err = -EINVAL;
900  u8 intrctrl = 0;
901 
902  if (!strcmp(ioat_interrupt_style, "msix"))
903  goto msix;
904  if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
905  goto msix_single_vector;
906  if (!strcmp(ioat_interrupt_style, "msi"))
907  goto msi;
908  if (!strcmp(ioat_interrupt_style, "intx"))
909  goto intx;
910  dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
911  goto err_no_irq;
912 
913 msix:
914  /* The number of MSI-X vectors should equal the number of channels */
915  msixcnt = device->common.chancnt;
916  for (i = 0; i < msixcnt; i++)
917  device->msix_entries[i].entry = i;
918 
919  err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
920  if (err < 0)
921  goto msi;
922  if (err > 0)
923  goto msix_single_vector;
924 
925  for (i = 0; i < msixcnt; i++) {
926  msix = &device->msix_entries[i];
927  chan = ioat_chan_by_index(device, i);
928  err = devm_request_irq(dev, msix->vector,
929  ioat_dma_do_interrupt_msix, 0,
930  "ioat-msix", chan);
931  if (err) {
932  for (j = 0; j < i; j++) {
933  msix = &device->msix_entries[j];
934  chan = ioat_chan_by_index(device, j);
935  devm_free_irq(dev, msix->vector, chan);
936  }
937  goto msix_single_vector;
938  }
939  }
941  goto done;
942 
943 msix_single_vector:
944  msix = &device->msix_entries[0];
945  msix->entry = 0;
946  err = pci_enable_msix(pdev, device->msix_entries, 1);
947  if (err)
948  goto msi;
949 
950  err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
951  "ioat-msix", device);
952  if (err) {
953  pci_disable_msix(pdev);
954  goto msi;
955  }
956  goto done;
957 
958 msi:
959  err = pci_enable_msi(pdev);
960  if (err)
961  goto intx;
962 
963  err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
964  "ioat-msi", device);
965  if (err) {
966  pci_disable_msi(pdev);
967  goto intx;
968  }
969  goto done;
970 
971 intx:
972  err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
973  IRQF_SHARED, "ioat-intx", device);
974  if (err)
975  goto err_no_irq;
976 
977 done:
978  if (device->intr_quirk)
979  device->intr_quirk(device);
980  intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
981  writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
982  return 0;
983 
984 err_no_irq:
985  /* Disable all interrupt generation */
986  writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
987  dev_err(dev, "no usable interrupts\n");
988  return err;
989 }
990 
991 static void ioat_disable_interrupts(struct ioatdma_device *device)
992 {
993  /* Disable all interrupt generation */
994  writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
995 }
996 
997 int __devinit ioat_probe(struct ioatdma_device *device)
998 {
999  int err = -ENODEV;
1000  struct dma_device *dma = &device->common;
1001  struct pci_dev *pdev = device->pdev;
1002  struct device *dev = &pdev->dev;
1003 
1004  /* DMA coherent memory pool for DMA descriptor allocations */
1005  device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1006  sizeof(struct ioat_dma_descriptor),
1007  64, 0);
1008  if (!device->dma_pool) {
1009  err = -ENOMEM;
1010  goto err_dma_pool;
1011  }
1012 
1013  device->completion_pool = pci_pool_create("completion_pool", pdev,
1014  sizeof(u64), SMP_CACHE_BYTES,
1015  SMP_CACHE_BYTES);
1016 
1017  if (!device->completion_pool) {
1018  err = -ENOMEM;
1019  goto err_completion_pool;
1020  }
1021 
1022  device->enumerate_channels(device);
1023 
1025  dma->dev = &pdev->dev;
1026 
1027  if (!dma->chancnt) {
1028  dev_err(dev, "channel enumeration error\n");
1029  goto err_setup_interrupts;
1030  }
1031 
1032  err = ioat_dma_setup_interrupts(device);
1033  if (err)
1034  goto err_setup_interrupts;
1035 
1036  err = device->self_test(device);
1037  if (err)
1038  goto err_self_test;
1039 
1040  return 0;
1041 
1042 err_self_test:
1043  ioat_disable_interrupts(device);
1044 err_setup_interrupts:
1045  pci_pool_destroy(device->completion_pool);
1046 err_completion_pool:
1047  pci_pool_destroy(device->dma_pool);
1048 err_dma_pool:
1049  return err;
1050 }
1051 
1052 int __devinit ioat_register(struct ioatdma_device *device)
1053 {
1054  int err = dma_async_device_register(&device->common);
1055 
1056  if (err) {
1057  ioat_disable_interrupts(device);
1058  pci_pool_destroy(device->completion_pool);
1059  pci_pool_destroy(device->dma_pool);
1060  }
1061 
1062  return err;
1063 }
1064 
1065 /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1066 static void ioat1_intr_quirk(struct ioatdma_device *device)
1067 {
1068  struct pci_dev *pdev = device->pdev;
1069  u32 dmactrl;
1070 
1071  pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1072  if (pdev->msi_enabled)
1073  dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1074  else
1075  dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1076  pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1077 }
1078 
1079 static ssize_t ring_size_show(struct dma_chan *c, char *page)
1080 {
1081  struct ioat_dma_chan *ioat = to_ioat_chan(c);
1082 
1083  return sprintf(page, "%d\n", ioat->desccount);
1084 }
1085 static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
1086 
1087 static ssize_t ring_active_show(struct dma_chan *c, char *page)
1088 {
1089  struct ioat_dma_chan *ioat = to_ioat_chan(c);
1090 
1091  return sprintf(page, "%d\n", ioat->active);
1092 }
1093 static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
1094 
1095 static ssize_t cap_show(struct dma_chan *c, char *page)
1096 {
1097  struct dma_device *dma = c->device;
1098 
1099  return sprintf(page, "copy%s%s%s%s%s%s\n",
1100  dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
1101  dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
1102  dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
1103  dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
1104  dma_has_cap(DMA_MEMSET, dma->cap_mask) ? " fill" : "",
1105  dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
1106 
1107 }
1109 
1110 static ssize_t version_show(struct dma_chan *c, char *page)
1111 {
1112  struct dma_device *dma = c->device;
1113  struct ioatdma_device *device = to_ioatdma_device(dma);
1114 
1115  return sprintf(page, "%d.%d\n",
1116  device->version >> 4, device->version & 0xf);
1117 }
1119 
1120 static struct attribute *ioat1_attrs[] = {
1121  &ring_size_attr.attr,
1122  &ring_active_attr.attr,
1123  &ioat_cap_attr.attr,
1124  &ioat_version_attr.attr,
1125  NULL,
1126 };
1127 
1128 static ssize_t
1129 ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
1130 {
1131  struct ioat_sysfs_entry *entry;
1132  struct ioat_chan_common *chan;
1133 
1134  entry = container_of(attr, struct ioat_sysfs_entry, attr);
1135  chan = container_of(kobj, struct ioat_chan_common, kobj);
1136 
1137  if (!entry->show)
1138  return -EIO;
1139  return entry->show(&chan->common, page);
1140 }
1141 
1142 const struct sysfs_ops ioat_sysfs_ops = {
1143  .show = ioat_attr_show,
1144 };
1145 
1146 static struct kobj_type ioat1_ktype = {
1147  .sysfs_ops = &ioat_sysfs_ops,
1148  .default_attrs = ioat1_attrs,
1149 };
1150 
1151 void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
1152 {
1153  struct dma_device *dma = &device->common;
1154  struct dma_chan *c;
1155 
1157  struct ioat_chan_common *chan = to_chan_common(c);
1158  struct kobject *parent = &c->dev->device.kobj;
1159  int err;
1160 
1161  err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata");
1162  if (err) {
1163  dev_warn(to_dev(chan),
1164  "sysfs init error (%d), continuing...\n", err);
1165  kobject_put(&chan->kobj);
1167  }
1168  }
1169 }
1170 
1171 void ioat_kobject_del(struct ioatdma_device *device)
1172 {
1173  struct dma_device *dma = &device->common;
1174  struct dma_chan *c;
1175 
1177  struct ioat_chan_common *chan = to_chan_common(c);
1178 
1179  if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
1180  kobject_del(&chan->kobj);
1181  kobject_put(&chan->kobj);
1182  }
1183  }
1184 }
1185 
1186 int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
1187 {
1188  struct pci_dev *pdev = device->pdev;
1189  struct dma_device *dma;
1190  int err;
1191 
1192  device->intr_quirk = ioat1_intr_quirk;
1193  device->enumerate_channels = ioat1_enumerate_channels;
1194  device->self_test = ioat_dma_self_test;
1195  device->timer_fn = ioat1_timer_event;
1196  device->cleanup_fn = ioat1_cleanup_event;
1197  dma = &device->common;
1198  dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1199  dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1200  dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
1201  dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
1203 
1204  err = ioat_probe(device);
1205  if (err)
1206  return err;
1207  ioat_set_tcp_copy_break(4096);
1208  err = ioat_register(device);
1209  if (err)
1210  return err;
1211  ioat_kobject_add(device, &ioat1_ktype);
1212 
1213  if (dca)
1214  device->dca = ioat_dca_init(pdev, device->reg_base);
1215 
1216  return err;
1217 }
1218 
1219 void __devexit ioat_dma_remove(struct ioatdma_device *device)
1220 {
1221  struct dma_device *dma = &device->common;
1222 
1223  ioat_disable_interrupts(device);
1224 
1225  ioat_kobject_del(device);
1226 
1228 
1229  pci_pool_destroy(device->dma_pool);
1230  pci_pool_destroy(device->completion_pool);
1231 
1232  INIT_LIST_HEAD(&dma->channels);
1233 }