29 #include <linux/module.h>
30 #include <linux/slab.h>
31 #include <linux/pci.h>
37 #include <linux/prefetch.h>
43 #include "../dmaengine.h"
48 "high-water mark for pushing ioat descriptors (default: 4)");
52 static void ioat1_dma_start_null_desc(
struct ioat_dma_chan *ioat);
63 unsigned long attnstatus;
79 chan = ioat_chan_by_index(instance, bit);
92 static irqreturn_t ioat_dma_do_interrupt_msix(
int irq,
void *data)
106 unsigned long data = (
unsigned long) c;
112 dma_cookie_init(&chan->
common);
132 struct device *
dev = &device->
pdev->dev;
139 dev_warn(dev,
"(%d) exceeds max supported channels (%zu)\n",
144 xfercap_scale &= 0x1f;
145 xfercap = (xfercap_scale == 0 ? -1 : (1
UL << xfercap_scale));
146 dev_dbg(dev,
"%s: xfercap = %d\n", __func__, xfercap);
148 #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
149 if (i7300_idle_platform_probe(
NULL,
NULL, 1) == 0)
152 for (i = 0; i < dma->
chancnt; i++) {
183 static void ioat1_dma_memcpy_issue_pending(
struct dma_chan *chan)
189 __ioat1_dma_memcpy_issue_pending(ioat);
202 u32 chansts, chanerr;
209 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
241 cookie = dma_cookie_assign(tx);
249 chain_tail->
hw->next = first->
txd.phys;
260 __ioat1_dma_memcpy_issue_pending(ioat);
279 ioatdma_device = ioat->
base.device;
280 desc = pci_pool_alloc(ioatdma_device->
dma_pool, flags, &phys);
284 desc_sw = kzalloc(
sizeof(*desc_sw), flags);
286 pci_pool_free(ioatdma_device->
dma_pool, desc, phys);
290 memset(desc, 0,
sizeof(*desc));
292 INIT_LIST_HEAD(&desc_sw->
tx_list);
294 desc_sw->
txd.tx_submit = ioat1_tx_submit;
302 static int ioat_initial_desc_count = 256;
305 "ioat1: initial descriptors per channel (default: 256)");
310 static int ioat1_dma_alloc_chan_resources(
struct dma_chan *c)
333 for (i = 0; i < ioat_initial_desc_count; i++) {
334 desc = ioat_dma_alloc_descriptor(ioat,
GFP_KERNEL);
344 list_splice(&tmp_list, &ioat->
free_desc);
358 ioat1_dma_start_null_desc(ioat);
368 static void ioat1_dma_free_chan_resources(
struct dma_chan *c)
372 struct ioatdma_device *ioatdma_device = chan->
device;
374 int in_use_descs = 0;
400 pci_pool_free(ioatdma_device->
dma_pool, desc->
hw,
407 pci_pool_free(ioatdma_device->
dma_pool, desc->
hw,
418 if (in_use_descs > 1)
446 new = ioat_dma_alloc_descriptor(ioat,
GFP_ATOMIC);
460 dma_addr_t dma_src,
size_t len,
unsigned long flags)
473 desc = ioat1_dma_get_next_descriptor(ioat);
495 async_tx_ack(&desc->
txd);
496 next = ioat1_dma_get_next_descriptor(ioat);
497 hw->
next = next ? next->
txd.phys : 0;
508 "chan%d - get_next_desc failed\n",
chan_num(chan));
519 hw->
ctl_f.compl_write = 1;
526 static void ioat1_cleanup_event(
unsigned long data)
541 ioat_unmap(pdev, hw->
dst_addr - offset, len,
545 ioat_unmap(pdev, hw->
src_addr - offset, len,
555 phys_complete = ioat_chansts_to_addr(completion);
557 dev_dbg(
to_dev(chan),
"%s: phys_complete: %#llx\n", __func__,
558 (
unsigned long long) phys_complete);
560 if (is_ioat_halted(completion)) {
568 return phys_complete;
590 __func__, (
unsigned long long) phys_complete);
604 dma_cookie_complete(tx);
613 if (tx->
phys != phys_complete) {
618 if (async_tx_test_ack(tx))
631 "%s cancel completion timeout\n",
667 if (!spin_trylock_bh(&ioat->
desc_lock)) {
672 __cleanup(ioat, phys_complete);
678 static void ioat1_timer_event(
unsigned long data)
693 ioat_set_chainaddr(ioat, desc->
txd.phys);
709 __cleanup(ioat, phys_complete);
711 ioat1_reset_channel(ioat);
716 if (ioat_chansts_to_addr(status) != 0)
732 struct ioatdma_device *device = chan->
device;
735 ret = dma_cookie_status(c, cookie, txstate);
741 return dma_cookie_status(c, cookie, txstate);
744 static void ioat1_dma_start_null_desc(
struct ioat_dma_chan *ioat)
752 desc = ioat1_dma_get_next_descriptor(ioat);
756 "Unable to start null desc - get next desc failed\n");
764 hw->
ctl_f.int_en = 1;
765 hw->
ctl_f.compl_write = 1;
770 async_tx_ack(&desc->
txd);
775 ioat_set_chainaddr(ioat, desc->
txd.phys);
783 #define IOAT_TEST_SIZE 2000
785 static void __devinit ioat_dma_test_callback(
void *dma_async_param)
802 struct device *dev = &device->
pdev->dev;
815 dest = kzalloc(
sizeof(
u8) * IOAT_TEST_SIZE,
GFP_KERNEL);
829 dev_err(dev,
"selftest cannot allocate chan resource\n");
838 tx = device->
common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
839 IOAT_TEST_SIZE, flags);
841 dev_err(dev,
"Self-test prep failed, disabling\n");
847 init_completion(&cmp);
848 tx->
callback = ioat_dma_test_callback;
852 dev_err(dev,
"Self-test setup failed, disabling\n");
863 dev_err(dev,
"Self-test copy timed out, disabling\n");
867 if (
memcmp(src, dest, IOAT_TEST_SIZE)) {
868 dev_err(dev,
"Self-test copy failed compare, disabling\n");
881 static char ioat_interrupt_style[32] =
"msix";
883 sizeof(ioat_interrupt_style), 0644);
885 "set ioat interrupt style: msix (default), "
886 "msix-single-vector, msi, intx)");
892 static int ioat_dma_setup_interrupts(
struct ioatdma_device *device)
896 struct device *dev = &pdev->
dev;
897 struct msix_entry *msix;
902 if (!
strcmp(ioat_interrupt_style,
"msix"))
904 if (!
strcmp(ioat_interrupt_style,
"msix-single-vector"))
905 goto msix_single_vector;
906 if (!
strcmp(ioat_interrupt_style,
"msi"))
908 if (!
strcmp(ioat_interrupt_style,
"intx"))
910 dev_err(dev,
"invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
915 msixcnt = device->
common.chancnt;
916 for (i = 0; i < msixcnt; i++)
923 goto msix_single_vector;
925 for (i = 0; i < msixcnt; i++) {
927 chan = ioat_chan_by_index(device, i);
928 err = devm_request_irq(dev, msix->vector,
929 ioat_dma_do_interrupt_msix, 0,
932 for (j = 0; j <
i; j++) {
934 chan = ioat_chan_by_index(device, j);
937 goto msix_single_vector;
950 err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
951 "ioat-msix", device);
959 err = pci_enable_msi(pdev);
963 err = devm_request_irq(dev, pdev->
irq, ioat_dma_do_interrupt, 0,
972 err = devm_request_irq(dev, pdev->
irq, ioat_dma_do_interrupt,
987 dev_err(dev,
"no usable interrupts\n");
991 static void ioat_disable_interrupts(
struct ioatdma_device *device)
1002 struct device *dev = &pdev->
dev;
1005 device->
dma_pool = pci_pool_create(
"dma_desc_pool", pdev,
1019 goto err_completion_pool;
1028 dev_err(dev,
"channel enumeration error\n");
1029 goto err_setup_interrupts;
1032 err = ioat_dma_setup_interrupts(device);
1034 goto err_setup_interrupts;
1043 ioat_disable_interrupts(device);
1044 err_setup_interrupts:
1046 err_completion_pool:
1047 pci_pool_destroy(device->
dma_pool);
1057 ioat_disable_interrupts(device);
1059 pci_pool_destroy(device->
dma_pool);
1066 static void ioat1_intr_quirk(
struct ioatdma_device *device)
1099 return sprintf(page,
"copy%s%s%s%s%s%s\n",
1115 return sprintf(page,
"%d.%d\n",
1120 static struct attribute *ioat1_attrs[] = {
1121 &ring_size_attr.
attr,
1122 &ring_active_attr.
attr,
1123 &ioat_cap_attr.
attr,
1124 &ioat_version_attr.
attr,
1143 .show = ioat_attr_show,
1148 .default_attrs = ioat1_attrs,
1164 "sysfs init error (%d), continuing...\n", err);
1195 device->
timer_fn = ioat1_timer_event;
1207 ioat_set_tcp_copy_break(4096);
1223 ioat_disable_interrupts(device);
1229 pci_pool_destroy(device->
dma_pool);