59 #include <linux/pci.h>
63 #include <linux/prefetch.h>
64 #include "../dmaengine.h"
71 #define src_cnt_to_sw(x) ((x) + 2)
72 #define src_cnt_to_hw(x) ((x) - 2)
77 static const u8 xor_idx_to_desc = 0xe0;
78 static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 };
79 static const u8 pq_idx_to_desc = 0xf8;
80 static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
86 return raw->
field[xor_idx_to_field[
idx]];
101 return raw->
field[pq_idx_to_field[
idx]];
120 size_t offset = len - desc->
hw->size;
124 switch (desc->
hw->ctl_f.op) {
126 if (!desc->
hw->ctl_f.null)
133 ioat_unmap(pdev, hw->
dst_addr - offset, len,
147 ext = ioat2_get_ring_ent(ioat, idx + 1);
154 for (i = 0; i < src_cnt; i++) {
157 ioat_unmap(pdev, src - offset, len,
163 ioat_unmap(pdev, xor->
dst_addr - offset, len,
170 ioat_unmap(pdev, xor->
dst_addr - offset, len,
184 ext = ioat2_get_ring_ent(ioat, idx + 1);
189 if (dmaf_p_disabled_continue(flags))
191 else if (dmaf_continue(flags))
194 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
197 for (i = 0; i < src_cnt; i++) {
200 ioat_unmap(pdev, src - offset, len,
207 ioat_unmap(pdev, pq->
p_addr - offset,
210 ioat_unmap(pdev, pq->
q_addr - offset,
217 if (!(flags & DMA_PREP_PQ_DISABLE_P))
218 ioat_unmap(pdev, pq->
p_addr - offset, len,
221 ioat_unmap(pdev, pq->
q_addr - offset, len,
227 dev_err(&pdev->
dev,
"%s: unknown op type: %#x\n",
228 __func__, desc->
hw->ctl_f.op);
264 bool seen_current =
false;
265 int idx = ioat->
tail,
i;
268 dev_dbg(
to_dev(chan),
"%s: head: %#x tail: %#x issued: %#x\n",
271 active = ioat2_ring_active(ioat);
272 for (i = 0; i < active && !seen_current; i++) {
276 prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));
277 desc = ioat2_get_ring_ent(ioat, idx + i);
281 dma_cookie_complete(tx);
282 ioat3_dma_unmap(ioat, desc, idx + i);
289 if (tx->
phys == phys_complete)
293 if (desc_has_ext(desc)) {
299 ioat->
tail = idx +
i;
300 BUG_ON(active && !seen_current);
303 if (active - i == 0) {
321 __cleanup(ioat, phys_complete);
325 static void ioat3_cleanup_event(
unsigned long data)
340 __cleanup(ioat, phys_complete);
345 static void ioat3_timer_event(
unsigned long data)
354 status = ioat_chansts(chan);
359 if (is_ioat_halted(status)) {
366 BUG_ON(is_ioat_bug(chanerr));
377 __cleanup(ioat, phys_complete);
380 ioat3_restart_channel(ioat);
395 active = ioat2_ring_active(ioat);
416 ret = dma_cookie_status(c, cookie, txstate);
422 return dma_cookie_status(c, cookie, txstate);
427 size_t len,
unsigned long flags)
433 u64 src_data = (0x0101010101010101ULL) * (value & 0xff);
434 int num_descs,
idx,
i;
436 num_descs = ioat2_xferlen_to_descs(ioat, len);
445 desc = ioat2_get_ring_ent(ioat, idx + i);
448 fill->
size = xfer_size;
457 }
while (++i < num_descs);
463 fill->
ctl_f.compl_write = 1;
473 size_t len,
unsigned long flags)
479 size_t total_len =
len;
483 int num_descs, with_ext,
idx,
i;
489 num_descs = ioat2_xferlen_to_descs(ioat, len);
514 desc = ioat2_get_ring_ent(ioat, idx + i);
521 ext = ioat2_get_ring_ent(ioat, idx + i + 1);
526 for (s = 0; s < src_cnt; s++)
527 xor_set_src(descs, src[s], offset, s);
528 xor->
size = xfer_size;
537 }
while ((i += 1 + with_ext) < num_descs);
547 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
553 hw->
ctl_f.compl_write = 1;
558 return &compl_desc->
txd;
563 unsigned int src_cnt,
size_t len,
unsigned long flags)
565 return __ioat3_prep_xor_lock(chan,
NULL, dest, src, src_cnt, len, flags);
570 unsigned int src_cnt,
size_t len,
578 return __ioat3_prep_xor_lock(chan, result, src[0], &src[1],
579 src_cnt - 1, len, flags);
592 dev_dbg(dev,
"desc[%d]: (%#llx->%#llx) flags: %#x"
593 " sz: %#x ctl: %#x (op: %d int: %d compl: %d pq: '%s%s' src_cnt: %d)\n",
594 desc_id(desc), (
unsigned long long) desc->
txd.phys,
595 (
unsigned long long) (pq_ex ? pq_ex->
next : pq->
next),
597 pq->
ctl_f.compl_write,
598 pq->
ctl_f.p_disable ?
"" :
"p", pq->
ctl_f.q_disable ?
"" :
"q",
600 for (i = 0; i < src_cnt; i++)
601 dev_dbg(dev,
"\tsrc[%d]: %#llx coef: %#x\n", i,
602 (
unsigned long long) pq_get_src(descs, i), pq->
coef[i]);
610 unsigned int src_cnt,
const unsigned char *scf,
611 size_t len,
unsigned long flags)
618 size_t total_len =
len;
624 int i,
s,
idx, with_ext, num_descs;
630 BUG_ON(src_cnt + dmaf_continue(flags) < 2);
632 num_descs = ioat2_xferlen_to_descs(ioat, len);
637 if (src_cnt + dmaf_p_disabled_continue(flags) > 3 ||
638 (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) {
659 desc = ioat2_get_ring_ent(ioat, idx + i);
666 ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);
672 for (s = 0; s < src_cnt; s++)
673 pq_set_src(descs, src[s], offset, scf[s], s);
676 if (dmaf_p_disabled_continue(flags))
677 pq_set_src(descs, dst[1], offset, 1, s++);
678 else if (dmaf_continue(flags)) {
679 pq_set_src(descs, dst[0], offset, 0, s++);
680 pq_set_src(descs, dst[1], offset, 1, s++);
681 pq_set_src(descs, dst[1], offset, 0, s++);
683 pq->
size = xfer_size;
694 }
while ((i += 1 + with_ext) < num_descs);
702 dump_pq_desc_dbg(ioat, desc, ext);
705 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
711 hw->
ctl_f.compl_write = 1;
716 return &compl_desc->
txd;
721 unsigned int src_cnt,
const unsigned char *scf,
size_t len,
725 if (flags & DMA_PREP_PQ_DISABLE_P)
733 if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) {
735 unsigned char single_source_coef[2];
737 BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
738 single_source[0] = src[0];
739 single_source[1] = src[0];
740 single_source_coef[0] = scf[0];
741 single_source_coef[1] = 0;
743 return __ioat3_prep_pq_lock(chan,
NULL, dst, single_source, 2,
744 single_source_coef, len, flags);
746 return __ioat3_prep_pq_lock(chan,
NULL, dst, src, src_cnt, scf,
752 unsigned int src_cnt,
const unsigned char *scf,
size_t len,
756 if (flags & DMA_PREP_PQ_DISABLE_P)
758 if (flags & DMA_PREP_PQ_DISABLE_Q)
766 return __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
772 unsigned int src_cnt,
size_t len,
unsigned long flags)
774 unsigned char scf[src_cnt];
782 return __ioat3_prep_pq_lock(chan,
NULL, pq, src, src_cnt, scf, len,
788 unsigned int src_cnt,
size_t len,
791 unsigned char scf[src_cnt];
804 return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf,
809 ioat3_prep_interrupt_lock(
struct dma_chan *c,
unsigned long flags)
816 desc = ioat2_get_ring_ent(ioat, ioat->
head);
823 hw->
ctl_f.int_en = 1;
825 hw->
ctl_f.compl_write = 1;
839 static void __devinit ioat3_dma_test_callback(
void *dma_async_param)
846 #define IOAT_NUM_SRC_TEST 6
864 struct device *dev = &device->
pdev->dev;
867 dev_dbg(dev,
"%s\n", __func__);
874 if (!xor_srcs[src_idx]) {
892 ptr[i] = (1 << src_idx);
896 cmp_byte ^= (
u8) (1 << src_idx);
898 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
899 (cmp_byte << 8) | cmp_byte;
920 dev_err(dev,
"Self-test xor prep failed\n");
926 init_completion(&cmp);
927 tx->
callback = ioat3_dma_test_callback;
931 dev_err(dev,
"Self-test xor setup failed\n");
940 dev_err(dev,
"Self-test xor timed out\n");
948 if (ptr[i] != cmp_word) {
949 dev_err(dev,
"Self-test xor failed compare\n");
962 xor_val_srcs[i] = xor_srcs[i];
963 xor_val_srcs[
i] =
dest;
967 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
974 dev_err(dev,
"Self-test zero prep failed\n");
980 init_completion(&cmp);
981 tx->
callback = ioat3_dma_test_callback;
985 dev_err(dev,
"Self-test zero setup failed\n");
994 dev_err(dev,
"Self-test validate timed out\n");
999 if (xor_val_result != 0) {
1000 dev_err(dev,
"Self-test validate failed compare\n");
1002 goto free_resources;
1007 goto free_resources;
1015 dev_err(dev,
"Self-test memset prep failed\n");
1017 goto free_resources;
1021 init_completion(&cmp);
1022 tx->
callback = ioat3_dma_test_callback;
1026 dev_err(dev,
"Self-test memset setup failed\n");
1028 goto free_resources;
1035 dev_err(dev,
"Self-test memset timed out\n");
1037 goto free_resources;
1043 dev_err(dev,
"Self-test memset failed compare\n");
1045 goto free_resources;
1051 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1058 dev_err(dev,
"Self-test 2nd zero prep failed\n");
1060 goto free_resources;
1064 init_completion(&cmp);
1065 tx->
callback = ioat3_dma_test_callback;
1069 dev_err(dev,
"Self-test 2nd zero setup failed\n");
1071 goto free_resources;
1078 dev_err(dev,
"Self-test 2nd validate timed out\n");
1080 goto free_resources;
1084 dev_err(dev,
"Self-test validate failed compare\n");
1086 goto free_resources;
1106 rc = ioat_xor_val_self_test(device);
1137 dev_err(&pdev->
dev,
"channel error register unreachable\n");
1152 static bool is_jf_ioat(
struct pci_dev *pdev)
1171 static bool is_snb_ioat(
struct pci_dev *pdev)
1197 bool is_raid_device =
false;
1203 device->
self_test = ioat3_dma_self_test;
1210 if (is_jf_ioat(pdev) || is_snb_ioat(pdev))
1223 is_raid_device =
true;
1234 is_raid_device =
true;
1235 dma_set_maxpq(dma, 8, 0);
1244 if (!(cap & IOAT_CAP_XOR)) {
1261 if (is_raid_device) {
1264 device->
timer_fn = ioat3_timer_event;
1271 #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
1276 #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
1284 ioat_set_tcp_copy_break(262144);
1287 chan = to_chan_common(c);