Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
dmatest.c
Go to the documentation of this file.
1 /*
2  * DMA Engine test module
3  *
4  * Copyright (C) 2007 Atmel Corporation
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 #include <linux/delay.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmaengine.h>
13 #include <linux/freezer.h>
14 #include <linux/init.h>
15 #include <linux/kthread.h>
16 #include <linux/module.h>
17 #include <linux/moduleparam.h>
18 #include <linux/random.h>
19 #include <linux/slab.h>
20 #include <linux/wait.h>
21 
22 static unsigned int test_buf_size = 16384;
23 module_param(test_buf_size, uint, S_IRUGO);
24 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
25 
26 static char test_channel[20];
27 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
28 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
29 
30 static char test_device[20];
31 module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
32 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
33 
34 static unsigned int threads_per_chan = 1;
35 module_param(threads_per_chan, uint, S_IRUGO);
36 MODULE_PARM_DESC(threads_per_chan,
37  "Number of threads to start per channel (default: 1)");
38 
39 static unsigned int max_channels;
40 module_param(max_channels, uint, S_IRUGO);
41 MODULE_PARM_DESC(max_channels,
42  "Maximum number of channels to use (default: all)");
43 
44 static unsigned int iterations;
45 module_param(iterations, uint, S_IRUGO);
46 MODULE_PARM_DESC(iterations,
47  "Iterations before stopping test (default: infinite)");
48 
49 static unsigned int xor_sources = 3;
50 module_param(xor_sources, uint, S_IRUGO);
51 MODULE_PARM_DESC(xor_sources,
52  "Number of xor source buffers (default: 3)");
53 
54 static unsigned int pq_sources = 3;
55 module_param(pq_sources, uint, S_IRUGO);
56 MODULE_PARM_DESC(pq_sources,
57  "Number of p+q source buffers (default: 3)");
58 
59 static int timeout = 3000;
60 module_param(timeout, uint, S_IRUGO);
61 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
62  "Pass -1 for infinite timeout");
63 
64 /*
65  * Initialization patterns. All bytes in the source buffer has bit 7
66  * set, all bytes in the destination buffer has bit 7 cleared.
67  *
68  * Bit 6 is set for all bytes which are to be copied by the DMA
69  * engine. Bit 5 is set for all bytes which are to be overwritten by
70  * the DMA engine.
71  *
72  * The remaining bits are the inverse of a counter which increments by
73  * one for each byte address.
74  */
75 #define PATTERN_SRC 0x80
76 #define PATTERN_DST 0x00
77 #define PATTERN_COPY 0x40
78 #define PATTERN_OVERWRITE 0x20
79 #define PATTERN_COUNT_MASK 0x1f
80 
82  struct list_head node;
83  struct task_struct *task;
84  struct dma_chan *chan;
85  u8 **srcs;
86  u8 **dsts;
88 };
89 
90 struct dmatest_chan {
91  struct list_head node;
92  struct dma_chan *chan;
94 };
95 
96 /*
97  * These are protected by dma_list_mutex since they're only used by
98  * the DMA filter function callback
99  */
100 static LIST_HEAD(dmatest_channels);
101 static unsigned int nr_channels;
102 
103 static bool dmatest_match_channel(struct dma_chan *chan)
104 {
105  if (test_channel[0] == '\0')
106  return true;
107  return strcmp(dma_chan_name(chan), test_channel) == 0;
108 }
109 
110 static bool dmatest_match_device(struct dma_device *device)
111 {
112  if (test_device[0] == '\0')
113  return true;
114  return strcmp(dev_name(device->dev), test_device) == 0;
115 }
116 
117 static unsigned long dmatest_random(void)
118 {
119  unsigned long buf;
120 
121  get_random_bytes(&buf, sizeof(buf));
122  return buf;
123 }
124 
125 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len)
126 {
127  unsigned int i;
128  u8 *buf;
129 
130  for (; (buf = *bufs); bufs++) {
131  for (i = 0; i < start; i++)
132  buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
133  for ( ; i < start + len; i++)
134  buf[i] = PATTERN_SRC | PATTERN_COPY
135  | (~i & PATTERN_COUNT_MASK);
136  for ( ; i < test_buf_size; i++)
137  buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
138  buf++;
139  }
140 }
141 
142 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len)
143 {
144  unsigned int i;
145  u8 *buf;
146 
147  for (; (buf = *bufs); bufs++) {
148  for (i = 0; i < start; i++)
149  buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
150  for ( ; i < start + len; i++)
151  buf[i] = PATTERN_DST | PATTERN_OVERWRITE
152  | (~i & PATTERN_COUNT_MASK);
153  for ( ; i < test_buf_size; i++)
154  buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
155  }
156 }
157 
158 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
159  unsigned int counter, bool is_srcbuf)
160 {
161  u8 diff = actual ^ pattern;
162  u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
163  const char *thread_name = current->comm;
164 
165  if (is_srcbuf)
166  pr_warning("%s: srcbuf[0x%x] overwritten!"
167  " Expected %02x, got %02x\n",
168  thread_name, index, expected, actual);
169  else if ((pattern & PATTERN_COPY)
170  && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
171  pr_warning("%s: dstbuf[0x%x] not copied!"
172  " Expected %02x, got %02x\n",
173  thread_name, index, expected, actual);
174  else if (diff & PATTERN_SRC)
175  pr_warning("%s: dstbuf[0x%x] was copied!"
176  " Expected %02x, got %02x\n",
177  thread_name, index, expected, actual);
178  else
179  pr_warning("%s: dstbuf[0x%x] mismatch!"
180  " Expected %02x, got %02x\n",
181  thread_name, index, expected, actual);
182 }
183 
184 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
185  unsigned int end, unsigned int counter, u8 pattern,
186  bool is_srcbuf)
187 {
188  unsigned int i;
189  unsigned int error_count = 0;
190  u8 actual;
191  u8 expected;
192  u8 *buf;
193  unsigned int counter_orig = counter;
194 
195  for (; (buf = *bufs); bufs++) {
196  counter = counter_orig;
197  for (i = start; i < end; i++) {
198  actual = buf[i];
199  expected = pattern | (~counter & PATTERN_COUNT_MASK);
200  if (actual != expected) {
201  if (error_count < 32)
202  dmatest_mismatch(actual, pattern, i,
203  counter, is_srcbuf);
204  error_count++;
205  }
206  counter++;
207  }
208  }
209 
210  if (error_count > 32)
211  pr_warning("%s: %u errors suppressed\n",
212  current->comm, error_count - 32);
213 
214  return error_count;
215 }
216 
217 /* poor man's completion - we want to use wait_event_freezable() on it */
218 struct dmatest_done {
219  bool done;
221 };
222 
223 static void dmatest_callback(void *arg)
224 {
225  struct dmatest_done *done = arg;
226 
227  done->done = true;
228  wake_up_all(done->wait);
229 }
230 
231 /*
232  * This function repeatedly tests DMA transfers of various lengths and
233  * offsets for a given operation type until it is told to exit by
234  * kthread_stop(). There may be multiple threads running this function
235  * in parallel for a single channel, and there may be multiple channels
236  * being tested in parallel.
237  *
238  * Before each test, the source and destination buffer is initialized
239  * with a known pattern. This pattern is different depending on
240  * whether it's in an area which is supposed to be copied or
241  * overwritten, and different in the source and destination buffers.
242  * So if the DMA engine doesn't copy exactly what we tell it to copy,
243  * we'll notice.
244  */
245 static int dmatest_func(void *data)
246 {
248  struct dmatest_thread *thread = data;
249  struct dmatest_done done = { .wait = &done_wait };
250  struct dma_chan *chan;
251  const char *thread_name;
252  unsigned int src_off, dst_off, len;
253  unsigned int error_count;
254  unsigned int failed_tests = 0;
255  unsigned int total_tests = 0;
257  enum dma_status status;
258  enum dma_ctrl_flags flags;
259  u8 pq_coefs[pq_sources + 1];
260  int ret;
261  int src_cnt;
262  int dst_cnt;
263  int i;
264 
265  thread_name = current->comm;
266  set_freezable();
267 
268  ret = -ENOMEM;
269 
270  smp_rmb();
271  chan = thread->chan;
272  if (thread->type == DMA_MEMCPY)
273  src_cnt = dst_cnt = 1;
274  else if (thread->type == DMA_XOR) {
275  src_cnt = xor_sources | 1; /* force odd to ensure dst = src */
276  dst_cnt = 1;
277  } else if (thread->type == DMA_PQ) {
278  src_cnt = pq_sources | 1; /* force odd to ensure dst = src */
279  dst_cnt = 2;
280  for (i = 0; i < src_cnt; i++)
281  pq_coefs[i] = 1;
282  } else
283  goto err_srcs;
284 
285  thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
286  if (!thread->srcs)
287  goto err_srcs;
288  for (i = 0; i < src_cnt; i++) {
289  thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL);
290  if (!thread->srcs[i])
291  goto err_srcbuf;
292  }
293  thread->srcs[i] = NULL;
294 
295  thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
296  if (!thread->dsts)
297  goto err_dsts;
298  for (i = 0; i < dst_cnt; i++) {
299  thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL);
300  if (!thread->dsts[i])
301  goto err_dstbuf;
302  }
303  thread->dsts[i] = NULL;
304 
305  set_user_nice(current, 10);
306 
307  /*
308  * src buffers are freed by the DMAEngine code with dma_unmap_single()
309  * dst buffers are freed by ourselves below
310  */
313 
314  while (!kthread_should_stop()
315  && !(iterations && total_tests >= iterations)) {
316  struct dma_device *dev = chan->device;
317  struct dma_async_tx_descriptor *tx = NULL;
318  dma_addr_t dma_srcs[src_cnt];
319  dma_addr_t dma_dsts[dst_cnt];
320  u8 align = 0;
321 
322  total_tests++;
323 
324  /* honor alignment restrictions */
325  if (thread->type == DMA_MEMCPY)
326  align = dev->copy_align;
327  else if (thread->type == DMA_XOR)
328  align = dev->xor_align;
329  else if (thread->type == DMA_PQ)
330  align = dev->pq_align;
331 
332  if (1 << align > test_buf_size) {
333  pr_err("%u-byte buffer too small for %d-byte alignment\n",
334  test_buf_size, 1 << align);
335  break;
336  }
337 
338  len = dmatest_random() % test_buf_size + 1;
339  len = (len >> align) << align;
340  if (!len)
341  len = 1 << align;
342  src_off = dmatest_random() % (test_buf_size - len + 1);
343  dst_off = dmatest_random() % (test_buf_size - len + 1);
344 
345  src_off = (src_off >> align) << align;
346  dst_off = (dst_off >> align) << align;
347 
348  dmatest_init_srcs(thread->srcs, src_off, len);
349  dmatest_init_dsts(thread->dsts, dst_off, len);
350 
351  for (i = 0; i < src_cnt; i++) {
352  u8 *buf = thread->srcs[i] + src_off;
353 
354  dma_srcs[i] = dma_map_single(dev->dev, buf, len,
355  DMA_TO_DEVICE);
356  }
357  /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
358  for (i = 0; i < dst_cnt; i++) {
359  dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
360  test_buf_size,
362  }
363 
364 
365  if (thread->type == DMA_MEMCPY)
366  tx = dev->device_prep_dma_memcpy(chan,
367  dma_dsts[0] + dst_off,
368  dma_srcs[0], len,
369  flags);
370  else if (thread->type == DMA_XOR)
371  tx = dev->device_prep_dma_xor(chan,
372  dma_dsts[0] + dst_off,
373  dma_srcs, src_cnt,
374  len, flags);
375  else if (thread->type == DMA_PQ) {
376  dma_addr_t dma_pq[dst_cnt];
377 
378  for (i = 0; i < dst_cnt; i++)
379  dma_pq[i] = dma_dsts[i] + dst_off;
380  tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
381  src_cnt, pq_coefs,
382  len, flags);
383  }
384 
385  if (!tx) {
386  for (i = 0; i < src_cnt; i++)
387  dma_unmap_single(dev->dev, dma_srcs[i], len,
388  DMA_TO_DEVICE);
389  for (i = 0; i < dst_cnt; i++)
390  dma_unmap_single(dev->dev, dma_dsts[i],
391  test_buf_size,
393  pr_warning("%s: #%u: prep error with src_off=0x%x "
394  "dst_off=0x%x len=0x%x\n",
395  thread_name, total_tests - 1,
396  src_off, dst_off, len);
397  msleep(100);
398  failed_tests++;
399  continue;
400  }
401 
402  done.done = false;
403  tx->callback = dmatest_callback;
404  tx->callback_param = &done;
405  cookie = tx->tx_submit(tx);
406 
407  if (dma_submit_error(cookie)) {
408  pr_warning("%s: #%u: submit error %d with src_off=0x%x "
409  "dst_off=0x%x len=0x%x\n",
410  thread_name, total_tests - 1, cookie,
411  src_off, dst_off, len);
412  msleep(100);
413  failed_tests++;
414  continue;
415  }
416  dma_async_issue_pending(chan);
417 
418  wait_event_freezable_timeout(done_wait, done.done,
419  msecs_to_jiffies(timeout));
420 
421  status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
422 
423  if (!done.done) {
424  /*
425  * We're leaving the timed out dma operation with
426  * dangling pointer to done_wait. To make this
427  * correct, we'll need to allocate wait_done for
428  * each test iteration and perform "who's gonna
429  * free it this time?" dancing. For now, just
430  * leave it dangling.
431  */
432  pr_warning("%s: #%u: test timed out\n",
433  thread_name, total_tests - 1);
434  failed_tests++;
435  continue;
436  } else if (status != DMA_SUCCESS) {
437  pr_warning("%s: #%u: got completion callback,"
438  " but status is \'%s\'\n",
439  thread_name, total_tests - 1,
440  status == DMA_ERROR ? "error" : "in progress");
441  failed_tests++;
442  continue;
443  }
444 
445  /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */
446  for (i = 0; i < dst_cnt; i++)
447  dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size,
449 
450  error_count = 0;
451 
452  pr_debug("%s: verifying source buffer...\n", thread_name);
453  error_count += dmatest_verify(thread->srcs, 0, src_off,
454  0, PATTERN_SRC, true);
455  error_count += dmatest_verify(thread->srcs, src_off,
456  src_off + len, src_off,
457  PATTERN_SRC | PATTERN_COPY, true);
458  error_count += dmatest_verify(thread->srcs, src_off + len,
459  test_buf_size, src_off + len,
460  PATTERN_SRC, true);
461 
462  pr_debug("%s: verifying dest buffer...\n",
463  thread->task->comm);
464  error_count += dmatest_verify(thread->dsts, 0, dst_off,
465  0, PATTERN_DST, false);
466  error_count += dmatest_verify(thread->dsts, dst_off,
467  dst_off + len, src_off,
468  PATTERN_SRC | PATTERN_COPY, false);
469  error_count += dmatest_verify(thread->dsts, dst_off + len,
470  test_buf_size, dst_off + len,
471  PATTERN_DST, false);
472 
473  if (error_count) {
474  pr_warning("%s: #%u: %u errors with "
475  "src_off=0x%x dst_off=0x%x len=0x%x\n",
476  thread_name, total_tests - 1, error_count,
477  src_off, dst_off, len);
478  failed_tests++;
479  } else {
480  pr_debug("%s: #%u: No errors with "
481  "src_off=0x%x dst_off=0x%x len=0x%x\n",
482  thread_name, total_tests - 1,
483  src_off, dst_off, len);
484  }
485  }
486 
487  ret = 0;
488  for (i = 0; thread->dsts[i]; i++)
489  kfree(thread->dsts[i]);
490 err_dstbuf:
491  kfree(thread->dsts);
492 err_dsts:
493  for (i = 0; thread->srcs[i]; i++)
494  kfree(thread->srcs[i]);
495 err_srcbuf:
496  kfree(thread->srcs);
497 err_srcs:
498  pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
499  thread_name, total_tests, failed_tests, ret);
500 
501  /* terminate all transfers on specified channels */
502  chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
503  if (iterations > 0)
504  while (!kthread_should_stop()) {
505  DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
506  interruptible_sleep_on(&wait_dmatest_exit);
507  }
508 
509  return ret;
510 }
511 
512 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
513 {
514  struct dmatest_thread *thread;
515  struct dmatest_thread *_thread;
516  int ret;
517 
518  list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
519  ret = kthread_stop(thread->task);
520  pr_debug("dmatest: thread %s exited with status %d\n",
521  thread->task->comm, ret);
522  list_del(&thread->node);
523  kfree(thread);
524  }
525 
526  /* terminate all transfers on specified channels */
527  dtc->chan->device->device_control(dtc->chan, DMA_TERMINATE_ALL, 0);
528 
529  kfree(dtc);
530 }
531 
532 static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type)
533 {
534  struct dmatest_thread *thread;
535  struct dma_chan *chan = dtc->chan;
536  char *op;
537  unsigned int i;
538 
539  if (type == DMA_MEMCPY)
540  op = "copy";
541  else if (type == DMA_XOR)
542  op = "xor";
543  else if (type == DMA_PQ)
544  op = "pq";
545  else
546  return -EINVAL;
547 
548  for (i = 0; i < threads_per_chan; i++) {
549  thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
550  if (!thread) {
551  pr_warning("dmatest: No memory for %s-%s%u\n",
552  dma_chan_name(chan), op, i);
553 
554  break;
555  }
556  thread->chan = dtc->chan;
557  thread->type = type;
558  smp_wmb();
559  thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
560  dma_chan_name(chan), op, i);
561  if (IS_ERR(thread->task)) {
562  pr_warning("dmatest: Failed to run thread %s-%s%u\n",
563  dma_chan_name(chan), op, i);
564  kfree(thread);
565  break;
566  }
567 
568  /* srcbuf and dstbuf are allocated by the thread itself */
569 
570  list_add_tail(&thread->node, &dtc->threads);
571  }
572 
573  return i;
574 }
575 
576 static int dmatest_add_channel(struct dma_chan *chan)
577 {
578  struct dmatest_chan *dtc;
579  struct dma_device *dma_dev = chan->device;
580  unsigned int thread_count = 0;
581  int cnt;
582 
583  dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
584  if (!dtc) {
585  pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
586  return -ENOMEM;
587  }
588 
589  dtc->chan = chan;
590  INIT_LIST_HEAD(&dtc->threads);
591 
592  if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
593  cnt = dmatest_add_threads(dtc, DMA_MEMCPY);
594  thread_count += cnt > 0 ? cnt : 0;
595  }
596  if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
597  cnt = dmatest_add_threads(dtc, DMA_XOR);
598  thread_count += cnt > 0 ? cnt : 0;
599  }
600  if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
601  cnt = dmatest_add_threads(dtc, DMA_PQ);
602  thread_count += cnt > 0 ? cnt : 0;
603  }
604 
605  pr_info("dmatest: Started %u threads using %s\n",
606  thread_count, dma_chan_name(chan));
607 
608  list_add_tail(&dtc->node, &dmatest_channels);
609  nr_channels++;
610 
611  return 0;
612 }
613 
614 static bool filter(struct dma_chan *chan, void *param)
615 {
616  if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device))
617  return false;
618  else
619  return true;
620 }
621 
622 static int __init dmatest_init(void)
623 {
625  struct dma_chan *chan;
626  int err = 0;
627 
628  dma_cap_zero(mask);
629  dma_cap_set(DMA_MEMCPY, mask);
630  for (;;) {
631  chan = dma_request_channel(mask, filter, NULL);
632  if (chan) {
633  err = dmatest_add_channel(chan);
634  if (err) {
635  dma_release_channel(chan);
636  break; /* add_channel failed, punt */
637  }
638  } else
639  break; /* no more channels available */
640  if (max_channels && nr_channels >= max_channels)
641  break; /* we have all we need */
642  }
643 
644  return err;
645 }
646 /* when compiled-in wait for drivers to load first */
647 late_initcall(dmatest_init);
648 
649 static void __exit dmatest_exit(void)
650 {
651  struct dmatest_chan *dtc, *_dtc;
652  struct dma_chan *chan;
653 
654  list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) {
655  list_del(&dtc->node);
656  chan = dtc->chan;
657  dmatest_cleanup_channel(dtc);
658  pr_debug("dmatest: dropped channel %s\n",
659  dma_chan_name(chan));
660  dma_release_channel(chan);
661  }
662 }
663 module_exit(dmatest_exit);
664 
665 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
666 MODULE_LICENSE("GPL v2");