Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
dmaengine.h
Go to the documentation of this file.
1 /*
2  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms of the GNU General Public License as published by the Free
6  * Software Foundation; either version 2 of the License, or (at your option)
7  * any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program; if not, write to the Free Software Foundation, Inc., 59
16  * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17  *
18  * The full GNU General Public License is included in this distribution in the
19  * file called COPYING.
20  */
21 #ifndef LINUX_DMAENGINE_H
22 #define LINUX_DMAENGINE_H
23 
24 #include <linux/device.h>
25 #include <linux/uio.h>
26 #include <linux/bug.h>
27 #include <linux/scatterlist.h>
28 #include <linux/bitmap.h>
29 #include <linux/types.h>
30 #include <asm/page.h>
31 
37 typedef s32 dma_cookie_t;
38 #define DMA_MIN_COOKIE 1
39 #define DMA_MAX_COOKIE INT_MAX
40 
41 #define dma_submit_error(cookie) ((cookie) < 0 ? 1 : 0)
42 
50 enum dma_status {
55 };
56 
77 /* last transaction type for creation of the capabilities mask */
79 };
80 
94 };
95 
128 struct data_chunk {
129  size_t size;
130  size_t icg;
131 };
132 
155  bool src_inc;
156  bool dst_inc;
157  bool src_sgl;
158  bool dst_sgl;
159  size_t numf;
160  size_t frame_size;
161  struct data_chunk sgl[0];
162 };
163 
187  DMA_PREP_INTERRUPT = (1 << 0),
188  DMA_CTRL_ACK = (1 << 1),
195  DMA_PREP_CONTINUE = (1 << 8),
196  DMA_PREP_FENCE = (1 << 9),
197 };
198 
219 };
220 
227 };
228 
237 };
238 
239 
245 
253  /* stats */
254  unsigned long memcpy_count;
255  unsigned long bytes_transferred;
256 };
257 
271 struct dma_chan {
275 
276  /* sysfs */
277  int chan_id;
278  struct dma_chan_dev *dev;
279 
284  void *private;
285 };
286 
294 struct dma_chan_dev {
295  struct dma_chan *chan;
296  struct device device;
297  int dev_id;
299 };
300 
311 };
312 
370  bool device_fc;
371  unsigned int slave_id;
372 };
373 
374 static inline const char *dma_chan_name(struct dma_chan *chan)
375 {
376  return dev_name(&chan->dev->device);
377 }
378 
379 void dma_chan_cleanup(struct kref *kref);
380 
392 typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param);
393 
394 typedef void (*dma_async_tx_callback)(void *dma_async_param);
414  enum dma_ctrl_flags flags; /* not a 'long' to pack with cookie */
416  struct dma_chan *chan;
420 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
424 #endif
425 };
426 
427 #ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
428 static inline void txd_lock(struct dma_async_tx_descriptor *txd)
429 {
430 }
431 static inline void txd_unlock(struct dma_async_tx_descriptor *txd)
432 {
433 }
434 static inline void txd_chain(struct dma_async_tx_descriptor *txd, struct dma_async_tx_descriptor *next)
435 {
436  BUG();
437 }
438 static inline void txd_clear_parent(struct dma_async_tx_descriptor *txd)
439 {
440 }
441 static inline void txd_clear_next(struct dma_async_tx_descriptor *txd)
442 {
443 }
444 static inline struct dma_async_tx_descriptor *txd_next(struct dma_async_tx_descriptor *txd)
445 {
446  return NULL;
447 }
448 static inline struct dma_async_tx_descriptor *txd_parent(struct dma_async_tx_descriptor *txd)
449 {
450  return NULL;
451 }
452 
453 #else
454 static inline void txd_lock(struct dma_async_tx_descriptor *txd)
455 {
456  spin_lock_bh(&txd->lock);
457 }
458 static inline void txd_unlock(struct dma_async_tx_descriptor *txd)
459 {
460  spin_unlock_bh(&txd->lock);
461 }
462 static inline void txd_chain(struct dma_async_tx_descriptor *txd, struct dma_async_tx_descriptor *next)
463 {
464  txd->next = next;
465  next->parent = txd;
466 }
467 static inline void txd_clear_parent(struct dma_async_tx_descriptor *txd)
468 {
469  txd->parent = NULL;
470 }
471 static inline void txd_clear_next(struct dma_async_tx_descriptor *txd)
472 {
473  txd->next = NULL;
474 }
475 static inline struct dma_async_tx_descriptor *txd_parent(struct dma_async_tx_descriptor *txd)
476 {
477  return txd->parent;
478 }
479 static inline struct dma_async_tx_descriptor *txd_next(struct dma_async_tx_descriptor *txd)
480 {
481  return txd->next;
482 }
483 #endif
484 
494 struct dma_tx_state {
498 };
499 
538 struct dma_device {
539 
540  unsigned int chancnt;
541  unsigned int privatecnt;
545  unsigned short max_xor;
546  unsigned short max_pq;
551  #define DMA_HAS_PQ_CONTINUE (1 << 15)
552 
553  int dev_id;
554  struct device *dev;
555 
558 
559  struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
560  struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
561  size_t len, unsigned long flags);
562  struct dma_async_tx_descriptor *(*device_prep_dma_xor)(
563  struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
564  unsigned int src_cnt, size_t len, unsigned long flags);
565  struct dma_async_tx_descriptor *(*device_prep_dma_xor_val)(
566  struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
567  size_t len, enum sum_check_flags *result, unsigned long flags);
568  struct dma_async_tx_descriptor *(*device_prep_dma_pq)(
569  struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
570  unsigned int src_cnt, const unsigned char *scf,
571  size_t len, unsigned long flags);
572  struct dma_async_tx_descriptor *(*device_prep_dma_pq_val)(
573  struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
574  unsigned int src_cnt, const unsigned char *scf, size_t len,
575  enum sum_check_flags *pqres, unsigned long flags);
576  struct dma_async_tx_descriptor *(*device_prep_dma_memset)(
577  struct dma_chan *chan, dma_addr_t dest, int value, size_t len,
578  unsigned long flags);
579  struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
580  struct dma_chan *chan, unsigned long flags);
581  struct dma_async_tx_descriptor *(*device_prep_dma_sg)(
582  struct dma_chan *chan,
583  struct scatterlist *dst_sg, unsigned int dst_nents,
584  struct scatterlist *src_sg, unsigned int src_nents,
585  unsigned long flags);
586 
587  struct dma_async_tx_descriptor *(*device_prep_slave_sg)(
588  struct dma_chan *chan, struct scatterlist *sgl,
589  unsigned int sg_len, enum dma_transfer_direction direction,
590  unsigned long flags, void *context);
591  struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)(
592  struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
593  size_t period_len, enum dma_transfer_direction direction,
594  unsigned long flags, void *context);
595  struct dma_async_tx_descriptor *(*device_prep_interleaved_dma)(
596  struct dma_chan *chan, struct dma_interleaved_template *xt,
597  unsigned long flags);
598  int (*device_control)(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
599  unsigned long arg);
600 
601  enum dma_status (*device_tx_status)(struct dma_chan *chan,
603  struct dma_tx_state *txstate);
605 };
606 
607 static inline int dmaengine_device_control(struct dma_chan *chan,
608  enum dma_ctrl_cmd cmd,
609  unsigned long arg)
610 {
611  return chan->device->device_control(chan, cmd, arg);
612 }
613 
614 static inline int dmaengine_slave_config(struct dma_chan *chan,
615  struct dma_slave_config *config)
616 {
617  return dmaengine_device_control(chan, DMA_SLAVE_CONFIG,
618  (unsigned long)config);
619 }
620 
621 static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
622  struct dma_chan *chan, dma_addr_t buf, size_t len,
623  enum dma_transfer_direction dir, unsigned long flags)
624 {
625  struct scatterlist sg;
626  sg_init_table(&sg, 1);
627  sg_dma_address(&sg) = buf;
628  sg_dma_len(&sg) = len;
629 
630  return chan->device->device_prep_slave_sg(chan, &sg, 1,
631  dir, flags, NULL);
632 }
633 
634 static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
635  struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
636  enum dma_transfer_direction dir, unsigned long flags)
637 {
638  return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
639  dir, flags, NULL);
640 }
641 
642 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
643 struct rio_dma_ext;
644 static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg(
645  struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
646  enum dma_transfer_direction dir, unsigned long flags,
647  struct rio_dma_ext *rio_ext)
648 {
649  return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
650  dir, flags, rio_ext);
651 }
652 #endif
653 
654 static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_cyclic(
655  struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
656  size_t period_len, enum dma_transfer_direction dir,
657  unsigned long flags)
658 {
659  return chan->device->device_prep_dma_cyclic(chan, buf_addr, buf_len,
660  period_len, dir, flags, NULL);
661 }
662 
663 static inline int dmaengine_terminate_all(struct dma_chan *chan)
664 {
665  return dmaengine_device_control(chan, DMA_TERMINATE_ALL, 0);
666 }
667 
668 static inline int dmaengine_pause(struct dma_chan *chan)
669 {
670  return dmaengine_device_control(chan, DMA_PAUSE, 0);
671 }
672 
673 static inline int dmaengine_resume(struct dma_chan *chan)
674 {
675  return dmaengine_device_control(chan, DMA_RESUME, 0);
676 }
677 
678 static inline enum dma_status dmaengine_tx_status(struct dma_chan *chan,
680 {
681  return chan->device->device_tx_status(chan, cookie, state);
682 }
683 
684 static inline dma_cookie_t dmaengine_submit(struct dma_async_tx_descriptor *desc)
685 {
686  return desc->tx_submit(desc);
687 }
688 
689 static inline bool dmaengine_check_align(u8 align, size_t off1, size_t off2, size_t len)
690 {
691  size_t mask;
692 
693  if (!align)
694  return true;
695  mask = (1 << align) - 1;
696  if (mask & (off1 | off2 | len))
697  return false;
698  return true;
699 }
700 
701 static inline bool is_dma_copy_aligned(struct dma_device *dev, size_t off1,
702  size_t off2, size_t len)
703 {
704  return dmaengine_check_align(dev->copy_align, off1, off2, len);
705 }
706 
707 static inline bool is_dma_xor_aligned(struct dma_device *dev, size_t off1,
708  size_t off2, size_t len)
709 {
710  return dmaengine_check_align(dev->xor_align, off1, off2, len);
711 }
712 
713 static inline bool is_dma_pq_aligned(struct dma_device *dev, size_t off1,
714  size_t off2, size_t len)
715 {
716  return dmaengine_check_align(dev->pq_align, off1, off2, len);
717 }
718 
719 static inline bool is_dma_fill_aligned(struct dma_device *dev, size_t off1,
720  size_t off2, size_t len)
721 {
722  return dmaengine_check_align(dev->fill_align, off1, off2, len);
723 }
724 
725 static inline void
726 dma_set_maxpq(struct dma_device *dma, int maxpq, int has_pq_continue)
727 {
728  dma->max_pq = maxpq;
729  if (has_pq_continue)
730  dma->max_pq |= DMA_HAS_PQ_CONTINUE;
731 }
732 
733 static inline bool dmaf_continue(enum dma_ctrl_flags flags)
734 {
735  return (flags & DMA_PREP_CONTINUE) == DMA_PREP_CONTINUE;
736 }
737 
738 static inline bool dmaf_p_disabled_continue(enum dma_ctrl_flags flags)
739 {
741 
742  return (flags & mask) == mask;
743 }
744 
745 static inline bool dma_dev_has_pq_continue(struct dma_device *dma)
746 {
748 }
749 
750 static inline unsigned short dma_dev_to_maxpq(struct dma_device *dma)
751 {
752  return dma->max_pq & ~DMA_HAS_PQ_CONTINUE;
753 }
754 
755 /* dma_maxpq - reduce maxpq in the face of continued operations
756  * @dma - dma device with PQ capability
757  * @flags - to check if DMA_PREP_CONTINUE and DMA_PREP_PQ_DISABLE_P are set
758  *
759  * When an engine does not support native continuation we need 3 extra
760  * source slots to reuse P and Q with the following coefficients:
761  * 1/ {00} * P : remove P from Q', but use it as a source for P'
762  * 2/ {01} * Q : use Q to continue Q' calculation
763  * 3/ {00} * Q : subtract Q from P' to cancel (2)
764  *
765  * In the case where P is disabled we only need 1 extra source:
766  * 1/ {01} * Q : use Q to continue Q' calculation
767  */
768 static inline int dma_maxpq(struct dma_device *dma, enum dma_ctrl_flags flags)
769 {
770  if (dma_dev_has_pq_continue(dma) || !dmaf_continue(flags))
771  return dma_dev_to_maxpq(dma);
772  else if (dmaf_p_disabled_continue(flags))
773  return dma_dev_to_maxpq(dma) - 1;
774  else if (dmaf_continue(flags))
775  return dma_dev_to_maxpq(dma) - 3;
776  BUG();
777 }
778 
779 /* --- public DMA engine API --- */
780 
781 #ifdef CONFIG_DMA_ENGINE
782 void dmaengine_get(void);
783 void dmaengine_put(void);
784 #else
785 static inline void dmaengine_get(void)
786 {
787 }
788 static inline void dmaengine_put(void)
789 {
790 }
791 #endif
792 
793 #ifdef CONFIG_NET_DMA
794 #define net_dmaengine_get() dmaengine_get()
795 #define net_dmaengine_put() dmaengine_put()
796 #else
797 static inline void net_dmaengine_get(void)
798 {
799 }
800 static inline void net_dmaengine_put(void)
801 {
802 }
803 #endif
804 
805 #ifdef CONFIG_ASYNC_TX_DMA
806 #define async_dmaengine_get() dmaengine_get()
807 #define async_dmaengine_put() dmaengine_put()
808 #ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
809 #define async_dma_find_channel(type) dma_find_channel(DMA_ASYNC_TX)
810 #else
811 #define async_dma_find_channel(type) dma_find_channel(type)
812 #endif /* CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH */
813 #else
814 static inline void async_dmaengine_get(void)
815 {
816 }
817 static inline void async_dmaengine_put(void)
818 {
819 }
820 static inline struct dma_chan *
821 async_dma_find_channel(enum dma_transaction_type type)
822 {
823  return NULL;
824 }
825 #endif /* CONFIG_ASYNC_TX_DMA */
826 
828  void *dest, void *src, size_t len);
830  struct page *page, unsigned int offset, void *kdata, size_t len);
832  struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
833  unsigned int src_off, size_t len);
835  struct dma_chan *chan);
836 
837 static inline void async_tx_ack(struct dma_async_tx_descriptor *tx)
838 {
839  tx->flags |= DMA_CTRL_ACK;
840 }
841 
842 static inline void async_tx_clear_ack(struct dma_async_tx_descriptor *tx)
843 {
844  tx->flags &= ~DMA_CTRL_ACK;
845 }
846 
847 static inline bool async_tx_test_ack(struct dma_async_tx_descriptor *tx)
848 {
849  return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;
850 }
851 
852 #define first_dma_cap(mask) __first_dma_cap(&(mask))
853 static inline int __first_dma_cap(const dma_cap_mask_t *srcp)
854 {
855  return min_t(int, DMA_TX_TYPE_END,
856  find_first_bit(srcp->bits, DMA_TX_TYPE_END));
857 }
858 
859 #define next_dma_cap(n, mask) __next_dma_cap((n), &(mask))
860 static inline int __next_dma_cap(int n, const dma_cap_mask_t *srcp)
861 {
862  return min_t(int, DMA_TX_TYPE_END,
863  find_next_bit(srcp->bits, DMA_TX_TYPE_END, n+1));
864 }
865 
866 #define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
867 static inline void
868 __dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
869 {
870  set_bit(tx_type, dstp->bits);
871 }
872 
873 #define dma_cap_clear(tx, mask) __dma_cap_clear((tx), &(mask))
874 static inline void
875 __dma_cap_clear(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
876 {
877  clear_bit(tx_type, dstp->bits);
878 }
879 
880 #define dma_cap_zero(mask) __dma_cap_zero(&(mask))
881 static inline void __dma_cap_zero(dma_cap_mask_t *dstp)
882 {
883  bitmap_zero(dstp->bits, DMA_TX_TYPE_END);
884 }
885 
886 #define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
887 static inline int
888 __dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
889 {
890  return test_bit(tx_type, srcp->bits);
891 }
892 
893 #define for_each_dma_cap_mask(cap, mask) \
894  for ((cap) = first_dma_cap(mask); \
895  (cap) < DMA_TX_TYPE_END; \
896  (cap) = next_dma_cap((cap), (mask)))
897 
905 static inline void dma_async_issue_pending(struct dma_chan *chan)
906 {
907  chan->device->device_issue_pending(chan);
908 }
909 
910 #define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan)
911 
923 static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
924  dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
925 {
926  struct dma_tx_state state;
927  enum dma_status status;
928 
929  status = chan->device->device_tx_status(chan, cookie, &state);
930  if (last)
931  *last = state.last;
932  if (used)
933  *used = state.used;
934  return status;
935 }
936 
937 #define dma_async_memcpy_complete(chan, cookie, last, used)\
938  dma_async_is_tx_complete(chan, cookie, last, used)
939 
949 static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie,
950  dma_cookie_t last_complete, dma_cookie_t last_used)
951 {
952  if (last_complete <= last_used) {
953  if ((cookie <= last_complete) || (cookie > last_used))
954  return DMA_SUCCESS;
955  } else {
956  if ((cookie <= last_complete) && (cookie > last_used))
957  return DMA_SUCCESS;
958  }
959  return DMA_IN_PROGRESS;
960 }
961 
962 static inline void
963 dma_set_tx_state(struct dma_tx_state *st, dma_cookie_t last, dma_cookie_t used, u32 residue)
964 {
965  if (st) {
966  st->last = last;
967  st->used = used;
968  st->residue = residue;
969  }
970 }
971 
972 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
973 #ifdef CONFIG_DMA_ENGINE
975 void dma_issue_pending_all(void);
976 struct dma_chan *__dma_request_channel(dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param);
977 void dma_release_channel(struct dma_chan *chan);
978 #else
979 static inline enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
980 {
981  return DMA_SUCCESS;
982 }
983 static inline void dma_issue_pending_all(void)
984 {
985 }
986 static inline struct dma_chan *__dma_request_channel(dma_cap_mask_t *mask,
987  dma_filter_fn fn, void *fn_param)
988 {
989  return NULL;
990 }
991 static inline void dma_release_channel(struct dma_chan *chan)
992 {
993 }
994 #endif
995 
996 /* --- DMA device --- */
997 
1001 struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type);
1002 struct dma_chan *net_dma_find_channel(void);
1003 #define dma_request_channel(mask, x, y) __dma_request_channel(&(mask), x, y)
1004 
1005 /* --- Helper iov-locking functions --- */
1006 
1010  struct page **pages;
1011 };
1012 
1016 };
1017 
1018 struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len);
1019 void dma_unpin_iovec_pages(struct dma_pinned_list* pinned_list);
1020 
1021 dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
1022  struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len);
1023 dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
1024  struct dma_pinned_list *pinned_list, struct page *page,
1025  unsigned int offset, size_t len);
1026 
1027 #endif /* DMAENGINE_H */