Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
ste_dma40.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) Ericsson AB 2007-2008
3  * Copyright (C) ST-Ericsson SA 2008-2010
4  * Author: Per Forlin <[email protected]> for ST-Ericsson
5  * Author: Jonas Aaberg <[email protected]> for ST-Ericsson
6  * License terms: GNU General Public License (GPL) version 2
7  */
8 
9 #include <linux/dma-mapping.h>
10 #include <linux/kernel.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/dmaengine.h>
14 #include <linux/platform_device.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/pm.h>
18 #include <linux/pm_runtime.h>
19 #include <linux/err.h>
20 #include <linux/amba/bus.h>
22 
23 #include <plat/ste_dma40.h>
24 
25 #include "dmaengine.h"
26 #include "ste_dma40_ll.h"
27 
28 #define D40_NAME "dma40"
29 
30 #define D40_PHY_CHAN -1
31 
32 /* For masking out/in 2 bit channel positions */
33 #define D40_CHAN_POS(chan) (2 * (chan / 2))
34 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
35 
36 /* Maximum iterations taken before giving up suspending a channel */
37 #define D40_SUSPEND_MAX_IT 500
38 
39 /* Milliseconds */
40 #define DMA40_AUTOSUSPEND_DELAY 100
41 
42 /* Hardware requirement on LCLA alignment */
43 #define LCLA_ALIGNMENT 0x40000
44 
45 /* Max number of links per event group */
46 #define D40_LCLA_LINK_PER_EVENT_GRP 128
47 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
48 
49 /* Attempts before giving up to trying to get pages that are aligned */
50 #define MAX_LCLA_ALLOC_ATTEMPTS 256
51 
52 /* Bit markings for allocation map */
53 #define D40_ALLOC_FREE (1 << 31)
54 #define D40_ALLOC_PHY (1 << 30)
55 #define D40_ALLOC_LOG_FREE 0
56 
70 };
71 
72 /*
73  * enum d40_events - The different Event Enables for the event lines.
74  *
75  * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
76  * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
77  * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
78  * @D40_ROUND_EVENTLINE: Status check for event line.
79  */
80 
81 enum d40_events {
86 };
87 
88 /*
89  * These are the registers that has to be saved and later restored
90  * when the DMA hw is powered off.
91  * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
92  */
93 static u32 d40_backup_regs[] = {
100 };
101 
102 #define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
103 
104 /* TODO: Check if all these registers have to be saved/restored on dma40 v3 */
105 static u32 d40_backup_regs_v3[] = {
122 };
123 
124 #define BACKUP_REGS_SZ_V3 ARRAY_SIZE(d40_backup_regs_v3)
125 
126 static u32 d40_backup_regs_chan[] = {
135 };
136 
148 struct d40_lli_pool {
149  void *base;
150  int size;
152  /* Space for dst and src, plus an extra for padding */
153  u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
154 };
155 
175 struct d40_desc {
176  /* LLI physical */
178  /* LLI logical */
180 
182  int lli_len;
185 
187  struct list_head node;
188 
190  bool cyclic;
191 };
192 
205  void *base;
208  int pages;
210  struct d40_desc **alloc_map;
211 };
212 
226 struct d40_phy_res {
228  bool reserved;
229  int num;
232 };
233 
234 struct d40_base;
235 
266 struct d40_chan {
268  int log_num;
270  bool busy;
272  struct dma_chan chan;
277  struct list_head queue;
281  struct d40_base *base;
282  /* Default register configurations */
287  /* Runtime reconfiguration */
290 };
291 
336 struct d40_base {
339  struct device *dev;
341  u8 rev:4;
342  struct clk *clk;
345  int irq;
357  /* Physical half channels */
360  void *lcpa_base;
369 };
370 
383  bool is_error;
384  int offset;
385 };
386 
393 struct d40_reg_val {
394  unsigned int reg;
395  unsigned int val;
396 };
397 
398 static struct device *chan2dev(struct d40_chan *d40c)
399 {
400  return &d40c->chan.dev->device;
401 }
402 
403 static bool chan_is_physical(struct d40_chan *chan)
404 {
405  return chan->log_num == D40_PHY_CHAN;
406 }
407 
408 static bool chan_is_logical(struct d40_chan *chan)
409 {
410  return !chan_is_physical(chan);
411 }
412 
413 static void __iomem *chan_base(struct d40_chan *chan)
414 {
415  return chan->base->virtbase + D40_DREG_PCBASE +
416  chan->phy_chan->num * D40_DREG_PCDELTA;
417 }
418 
419 #define d40_err(dev, format, arg...) \
420  dev_err(dev, "[%s] " format, __func__, ## arg)
421 
422 #define chan_err(d40c, format, arg...) \
423  d40_err(chan2dev(d40c), format, ## arg)
424 
425 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
426  int lli_len)
427 {
428  bool is_log = chan_is_logical(d40c);
429  u32 align;
430  void *base;
431 
432  if (is_log)
433  align = sizeof(struct d40_log_lli);
434  else
435  align = sizeof(struct d40_phy_lli);
436 
437  if (lli_len == 1) {
438  base = d40d->lli_pool.pre_alloc_lli;
439  d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
440  d40d->lli_pool.base = NULL;
441  } else {
442  d40d->lli_pool.size = lli_len * 2 * align;
443 
444  base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
445  d40d->lli_pool.base = base;
446 
447  if (d40d->lli_pool.base == NULL)
448  return -ENOMEM;
449  }
450 
451  if (is_log) {
452  d40d->lli_log.src = PTR_ALIGN(base, align);
453  d40d->lli_log.dst = d40d->lli_log.src + lli_len;
454 
455  d40d->lli_pool.dma_addr = 0;
456  } else {
457  d40d->lli_phy.src = PTR_ALIGN(base, align);
458  d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
459 
460  d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
461  d40d->lli_phy.src,
462  d40d->lli_pool.size,
463  DMA_TO_DEVICE);
464 
465  if (dma_mapping_error(d40c->base->dev,
466  d40d->lli_pool.dma_addr)) {
467  kfree(d40d->lli_pool.base);
468  d40d->lli_pool.base = NULL;
469  d40d->lli_pool.dma_addr = 0;
470  return -ENOMEM;
471  }
472  }
473 
474  return 0;
475 }
476 
477 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
478 {
479  if (d40d->lli_pool.dma_addr)
480  dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
481  d40d->lli_pool.size, DMA_TO_DEVICE);
482 
483  kfree(d40d->lli_pool.base);
484  d40d->lli_pool.base = NULL;
485  d40d->lli_pool.size = 0;
486  d40d->lli_log.src = NULL;
487  d40d->lli_log.dst = NULL;
488  d40d->lli_phy.src = NULL;
489  d40d->lli_phy.dst = NULL;
490 }
491 
492 static int d40_lcla_alloc_one(struct d40_chan *d40c,
493  struct d40_desc *d40d)
494 {
495  unsigned long flags;
496  int i;
497  int ret = -EINVAL;
498  int p;
499 
500  spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
501 
502  p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
503 
504  /*
505  * Allocate both src and dst at the same time, therefore the half
506  * start on 1 since 0 can't be used since zero is used as end marker.
507  */
508  for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
509  if (!d40c->base->lcla_pool.alloc_map[p + i]) {
510  d40c->base->lcla_pool.alloc_map[p + i] = d40d;
511  d40d->lcla_alloc++;
512  ret = i;
513  break;
514  }
515  }
516 
517  spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
518 
519  return ret;
520 }
521 
522 static int d40_lcla_free_all(struct d40_chan *d40c,
523  struct d40_desc *d40d)
524 {
525  unsigned long flags;
526  int i;
527  int ret = -EINVAL;
528 
529  if (chan_is_physical(d40c))
530  return 0;
531 
532  spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
533 
534  for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
535  if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
536  D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
537  d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
539  d40d->lcla_alloc--;
540  if (d40d->lcla_alloc == 0) {
541  ret = 0;
542  break;
543  }
544  }
545  }
546 
547  spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
548 
549  return ret;
550 
551 }
552 
553 static void d40_desc_remove(struct d40_desc *d40d)
554 {
555  list_del(&d40d->node);
556 }
557 
558 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
559 {
560  struct d40_desc *desc = NULL;
561 
562  if (!list_empty(&d40c->client)) {
563  struct d40_desc *d;
564  struct d40_desc *_d;
565 
566  list_for_each_entry_safe(d, _d, &d40c->client, node) {
567  if (async_tx_test_ack(&d->txd)) {
568  d40_desc_remove(d);
569  desc = d;
570  memset(desc, 0, sizeof(*desc));
571  break;
572  }
573  }
574  }
575 
576  if (!desc)
577  desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
578 
579  if (desc)
580  INIT_LIST_HEAD(&desc->node);
581 
582  return desc;
583 }
584 
585 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
586 {
587 
588  d40_pool_lli_free(d40c, d40d);
589  d40_lcla_free_all(d40c, d40d);
590  kmem_cache_free(d40c->base->desc_slab, d40d);
591 }
592 
593 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
594 {
595  list_add_tail(&desc->node, &d40c->active);
596 }
597 
598 static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
599 {
600  struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
601  struct d40_phy_lli *lli_src = desc->lli_phy.src;
602  void __iomem *base = chan_base(chan);
603 
604  writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
605  writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
606  writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
607  writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
608 
609  writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
610  writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
611  writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
612  writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
613 }
614 
615 static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
616 {
617  struct d40_lcla_pool *pool = &chan->base->lcla_pool;
618  struct d40_log_lli_bidir *lli = &desc->lli_log;
619  int lli_current = desc->lli_current;
620  int lli_len = desc->lli_len;
621  bool cyclic = desc->cyclic;
622  int curr_lcla = -EINVAL;
623  int first_lcla = 0;
624  bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;
625  bool linkback;
626 
627  /*
628  * We may have partially running cyclic transfers, in case we did't get
629  * enough LCLA entries.
630  */
631  linkback = cyclic && lli_current == 0;
632 
633  /*
634  * For linkback, we need one LCLA even with only one link, because we
635  * can't link back to the one in LCPA space
636  */
637  if (linkback || (lli_len - lli_current > 1)) {
638  curr_lcla = d40_lcla_alloc_one(chan, desc);
639  first_lcla = curr_lcla;
640  }
641 
642  /*
643  * For linkback, we normally load the LCPA in the loop since we need to
644  * link it to the second LCLA and not the first. However, if we
645  * couldn't even get a first LCLA, then we have to run in LCPA and
646  * reload manually.
647  */
648  if (!linkback || curr_lcla == -EINVAL) {
649  unsigned int flags = 0;
650 
651  if (curr_lcla == -EINVAL)
652  flags |= LLI_TERM_INT;
653 
655  &lli->dst[lli_current],
656  &lli->src[lli_current],
657  curr_lcla,
658  flags);
659  lli_current++;
660  }
661 
662  if (curr_lcla < 0)
663  goto out;
664 
665  for (; lli_current < lli_len; lli_current++) {
666  unsigned int lcla_offset = chan->phy_chan->num * 1024 +
667  8 * curr_lcla * 2;
668  struct d40_log_lli *lcla = pool->base + lcla_offset;
669  unsigned int flags = 0;
670  int next_lcla;
671 
672  if (lli_current + 1 < lli_len)
673  next_lcla = d40_lcla_alloc_one(chan, desc);
674  else
675  next_lcla = linkback ? first_lcla : -EINVAL;
676 
677  if (cyclic || next_lcla == -EINVAL)
678  flags |= LLI_TERM_INT;
679 
680  if (linkback && curr_lcla == first_lcla) {
681  /* First link goes in both LCPA and LCLA */
683  &lli->dst[lli_current],
684  &lli->src[lli_current],
685  next_lcla, flags);
686  }
687 
688  /*
689  * One unused LCLA in the cyclic case if the very first
690  * next_lcla fails...
691  */
693  &lli->dst[lli_current],
694  &lli->src[lli_current],
695  next_lcla, flags);
696 
697  /*
698  * Cache maintenance is not needed if lcla is
699  * mapped in esram
700  */
701  if (!use_esram_lcla) {
703  pool->dma_addr, lcla_offset,
704  2 * sizeof(struct d40_log_lli),
705  DMA_TO_DEVICE);
706  }
707  curr_lcla = next_lcla;
708 
709  if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
710  lli_current++;
711  break;
712  }
713  }
714 
715 out:
716  desc->lli_current = lli_current;
717 }
718 
719 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
720 {
721  if (chan_is_physical(d40c)) {
722  d40_phy_lli_load(d40c, d40d);
723  d40d->lli_current = d40d->lli_len;
724  } else
725  d40_log_lli_to_lcxa(d40c, d40d);
726 }
727 
728 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
729 {
730  struct d40_desc *d;
731 
732  if (list_empty(&d40c->active))
733  return NULL;
734 
735  d = list_first_entry(&d40c->active,
736  struct d40_desc,
737  node);
738  return d;
739 }
740 
741 /* remove desc from current queue and add it to the pending_queue */
742 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
743 {
744  d40_desc_remove(desc);
745  desc->is_in_client_list = false;
746  list_add_tail(&desc->node, &d40c->pending_queue);
747 }
748 
749 static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
750 {
751  struct d40_desc *d;
752 
753  if (list_empty(&d40c->pending_queue))
754  return NULL;
755 
756  d = list_first_entry(&d40c->pending_queue,
757  struct d40_desc,
758  node);
759  return d;
760 }
761 
762 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
763 {
764  struct d40_desc *d;
765 
766  if (list_empty(&d40c->queue))
767  return NULL;
768 
769  d = list_first_entry(&d40c->queue,
770  struct d40_desc,
771  node);
772  return d;
773 }
774 
775 static int d40_psize_2_burst_size(bool is_log, int psize)
776 {
777  if (is_log) {
778  if (psize == STEDMA40_PSIZE_LOG_1)
779  return 1;
780  } else {
781  if (psize == STEDMA40_PSIZE_PHY_1)
782  return 1;
783  }
784 
785  return 2 << psize;
786 }
787 
788 /*
789  * The dma only supports transmitting packages up to
790  * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
791  * dma elements required to send the entire sg list
792  */
793 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
794 {
795  int dmalen;
796  u32 max_w = max(data_width1, data_width2);
797  u32 min_w = min(data_width1, data_width2);
798  u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
799 
800  if (seg_max > STEDMA40_MAX_SEG_SIZE)
801  seg_max -= (1 << max_w);
802 
803  if (!IS_ALIGNED(size, 1 << max_w))
804  return -EINVAL;
805 
806  if (size <= seg_max)
807  dmalen = 1;
808  else {
809  dmalen = size / seg_max;
810  if (dmalen * seg_max < size)
811  dmalen++;
812  }
813  return dmalen;
814 }
815 
816 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
817  u32 data_width1, u32 data_width2)
818 {
819  struct scatterlist *sg;
820  int i;
821  int len = 0;
822  int ret;
823 
824  for_each_sg(sgl, sg, sg_len, i) {
825  ret = d40_size_2_dmalen(sg_dma_len(sg),
826  data_width1, data_width2);
827  if (ret < 0)
828  return ret;
829  len += ret;
830  }
831  return len;
832 }
833 
834 
835 #ifdef CONFIG_PM
836 static void dma40_backup(void __iomem *baseaddr, u32 *backup,
837  u32 *regaddr, int num, bool save)
838 {
839  int i;
840 
841  for (i = 0; i < num; i++) {
842  void __iomem *addr = baseaddr + regaddr[i];
843 
844  if (save)
845  backup[i] = readl_relaxed(addr);
846  else
847  writel_relaxed(backup[i], addr);
848  }
849 }
850 
851 static void d40_save_restore_registers(struct d40_base *base, bool save)
852 {
853  int i;
854 
855  /* Save/Restore channel specific registers */
856  for (i = 0; i < base->num_phy_chans; i++) {
857  void __iomem *addr;
858  int idx;
859 
860  if (base->phy_res[i].reserved)
861  continue;
862 
863  addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
864  idx = i * ARRAY_SIZE(d40_backup_regs_chan);
865 
866  dma40_backup(addr, &base->reg_val_backup_chan[idx],
867  d40_backup_regs_chan,
868  ARRAY_SIZE(d40_backup_regs_chan),
869  save);
870  }
871 
872  /* Save/Restore global registers */
873  dma40_backup(base->virtbase, base->reg_val_backup,
874  d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
875  save);
876 
877  /* Save/Restore registers only existing on dma40 v3 and later */
878  if (base->rev >= 3)
879  dma40_backup(base->virtbase, base->reg_val_backup_v3,
880  d40_backup_regs_v3,
881  ARRAY_SIZE(d40_backup_regs_v3),
882  save);
883 }
884 #else
885 static void d40_save_restore_registers(struct d40_base *base, bool save)
886 {
887 }
888 #endif
889 
890 static int __d40_execute_command_phy(struct d40_chan *d40c,
891  enum d40_command command)
892 {
893  u32 status;
894  int i;
895  void __iomem *active_reg;
896  int ret = 0;
897  unsigned long flags;
898  u32 wmask;
899 
900  if (command == D40_DMA_STOP) {
901  ret = __d40_execute_command_phy(d40c, D40_DMA_SUSPEND_REQ);
902  if (ret)
903  return ret;
904  }
905 
906  spin_lock_irqsave(&d40c->base->execmd_lock, flags);
907 
908  if (d40c->phy_chan->num % 2 == 0)
909  active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
910  else
911  active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
912 
913  if (command == D40_DMA_SUSPEND_REQ) {
914  status = (readl(active_reg) &
915  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
916  D40_CHAN_POS(d40c->phy_chan->num);
917 
918  if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
919  goto done;
920  }
921 
922  wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
923  writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
924  active_reg);
925 
926  if (command == D40_DMA_SUSPEND_REQ) {
927 
928  for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
929  status = (readl(active_reg) &
930  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
931  D40_CHAN_POS(d40c->phy_chan->num);
932 
933  cpu_relax();
934  /*
935  * Reduce the number of bus accesses while
936  * waiting for the DMA to suspend.
937  */
938  udelay(3);
939 
940  if (status == D40_DMA_STOP ||
941  status == D40_DMA_SUSPENDED)
942  break;
943  }
944 
945  if (i == D40_SUSPEND_MAX_IT) {
946  chan_err(d40c,
947  "unable to suspend the chl %d (log: %d) status %x\n",
948  d40c->phy_chan->num, d40c->log_num,
949  status);
950  dump_stack();
951  ret = -EBUSY;
952  }
953 
954  }
955 done:
956  spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
957  return ret;
958 }
959 
960 static void d40_term_all(struct d40_chan *d40c)
961 {
962  struct d40_desc *d40d;
963  struct d40_desc *_d;
964 
965  /* Release active descriptors */
966  while ((d40d = d40_first_active_get(d40c))) {
967  d40_desc_remove(d40d);
968  d40_desc_free(d40c, d40d);
969  }
970 
971  /* Release queued descriptors waiting for transfer */
972  while ((d40d = d40_first_queued(d40c))) {
973  d40_desc_remove(d40d);
974  d40_desc_free(d40c, d40d);
975  }
976 
977  /* Release pending descriptors */
978  while ((d40d = d40_first_pending(d40c))) {
979  d40_desc_remove(d40d);
980  d40_desc_free(d40c, d40d);
981  }
982 
983  /* Release client owned descriptors */
984  if (!list_empty(&d40c->client))
985  list_for_each_entry_safe(d40d, _d, &d40c->client, node) {
986  d40_desc_remove(d40d);
987  d40_desc_free(d40c, d40d);
988  }
989 
990  /* Release descriptors in prepare queue */
991  if (!list_empty(&d40c->prepare_queue))
992  list_for_each_entry_safe(d40d, _d,
993  &d40c->prepare_queue, node) {
994  d40_desc_remove(d40d);
995  d40_desc_free(d40c, d40d);
996  }
997 
998  d40c->pending_tx = 0;
999 }
1000 
1001 static void __d40_config_set_event(struct d40_chan *d40c,
1003  int reg)
1004 {
1005  void __iomem *addr = chan_base(d40c) + reg;
1006  int tries;
1007  u32 status;
1008 
1009  switch (event_type) {
1010 
1012 
1014  | ~D40_EVENTLINE_MASK(event), addr);
1015  break;
1016 
1018  status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1019  D40_EVENTLINE_POS(event);
1020 
1021  if (status == D40_DEACTIVATE_EVENTLINE ||
1022  status == D40_SUSPEND_REQ_EVENTLINE)
1023  break;
1024 
1026  | ~D40_EVENTLINE_MASK(event), addr);
1027 
1028  for (tries = 0 ; tries < D40_SUSPEND_MAX_IT; tries++) {
1029 
1030  status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1031  D40_EVENTLINE_POS(event);
1032 
1033  cpu_relax();
1034  /*
1035  * Reduce the number of bus accesses while
1036  * waiting for the DMA to suspend.
1037  */
1038  udelay(3);
1039 
1040  if (status == D40_DEACTIVATE_EVENTLINE)
1041  break;
1042  }
1043 
1044  if (tries == D40_SUSPEND_MAX_IT) {
1045  chan_err(d40c,
1046  "unable to stop the event_line chl %d (log: %d)"
1047  "status %x\n", d40c->phy_chan->num,
1048  d40c->log_num, status);
1049  }
1050  break;
1051 
1053  /*
1054  * The hardware sometimes doesn't register the enable when src and dst
1055  * event lines are active on the same logical channel. Retry to ensure
1056  * it does. Usually only one retry is sufficient.
1057  */
1058  tries = 100;
1059  while (--tries) {
1061  D40_EVENTLINE_POS(event)) |
1062  ~D40_EVENTLINE_MASK(event), addr);
1063 
1064  if (readl(addr) & D40_EVENTLINE_MASK(event))
1065  break;
1066  }
1067 
1068  if (tries != 99)
1069  dev_dbg(chan2dev(d40c),
1070  "[%s] workaround enable S%cLNK (%d tries)\n",
1071  __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
1072  100 - tries);
1073 
1074  WARN_ON(!tries);
1075  break;
1076 
1077  case D40_ROUND_EVENTLINE:
1078  BUG();
1079  break;
1080 
1081  }
1082 }
1083 
1084 static void d40_config_set_event(struct d40_chan *d40c,
1085  enum d40_events event_type)
1086 {
1087  /* Enable event line connected to device (or memcpy) */
1088  if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
1089  (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
1090  u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1091 
1092  __d40_config_set_event(d40c, event_type, event,
1094  }
1095 
1096  if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
1097  u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1098 
1099  __d40_config_set_event(d40c, event_type, event,
1101  }
1102 }
1103 
1104 static u32 d40_chan_has_events(struct d40_chan *d40c)
1105 {
1106  void __iomem *chanbase = chan_base(d40c);
1107  u32 val;
1108 
1109  val = readl(chanbase + D40_CHAN_REG_SSLNK);
1110  val |= readl(chanbase + D40_CHAN_REG_SDLNK);
1111 
1112  return val;
1113 }
1114 
1115 static int
1116 __d40_execute_command_log(struct d40_chan *d40c, enum d40_command command)
1117 {
1118  unsigned long flags;
1119  int ret = 0;
1120  u32 active_status;
1121  void __iomem *active_reg;
1122 
1123  if (d40c->phy_chan->num % 2 == 0)
1124  active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1125  else
1126  active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1127 
1128 
1129  spin_lock_irqsave(&d40c->phy_chan->lock, flags);
1130 
1131  switch (command) {
1132  case D40_DMA_STOP:
1133  case D40_DMA_SUSPEND_REQ:
1134 
1135  active_status = (readl(active_reg) &
1136  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1137  D40_CHAN_POS(d40c->phy_chan->num);
1138 
1139  if (active_status == D40_DMA_RUN)
1140  d40_config_set_event(d40c, D40_SUSPEND_REQ_EVENTLINE);
1141  else
1142  d40_config_set_event(d40c, D40_DEACTIVATE_EVENTLINE);
1143 
1144  if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP))
1145  ret = __d40_execute_command_phy(d40c, command);
1146 
1147  break;
1148 
1149  case D40_DMA_RUN:
1150 
1151  d40_config_set_event(d40c, D40_ACTIVATE_EVENTLINE);
1152  ret = __d40_execute_command_phy(d40c, command);
1153  break;
1154 
1155  case D40_DMA_SUSPENDED:
1156  BUG();
1157  break;
1158  }
1159 
1160  spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
1161  return ret;
1162 }
1163 
1164 static int d40_channel_execute_command(struct d40_chan *d40c,
1165  enum d40_command command)
1166 {
1167  if (chan_is_logical(d40c))
1168  return __d40_execute_command_log(d40c, command);
1169  else
1170  return __d40_execute_command_phy(d40c, command);
1171 }
1172 
1173 static u32 d40_get_prmo(struct d40_chan *d40c)
1174 {
1175  static const unsigned int phy_map[] = {
1182  };
1183  static const unsigned int log_map[] = {
1190  };
1191 
1192  if (chan_is_physical(d40c))
1193  return phy_map[d40c->dma_cfg.mode_opt];
1194  else
1195  return log_map[d40c->dma_cfg.mode_opt];
1196 }
1197 
1198 static void d40_config_write(struct d40_chan *d40c)
1199 {
1200  u32 addr_base;
1201  u32 var;
1202 
1203  /* Odd addresses are even addresses + 4 */
1204  addr_base = (d40c->phy_chan->num % 2) * 4;
1205  /* Setup channel mode to logical or physical */
1206  var = ((u32)(chan_is_logical(d40c)) + 1) <<
1207  D40_CHAN_POS(d40c->phy_chan->num);
1208  writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
1209 
1210  /* Setup operational mode option register */
1211  var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
1212 
1213  writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
1214 
1215  if (chan_is_logical(d40c)) {
1216  int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
1218  void __iomem *chanbase = chan_base(d40c);
1219 
1220  /* Set default config for CFG reg */
1221  writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
1222  writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
1223 
1224  /* Set LIDX for lcla */
1225  writel(lidx, chanbase + D40_CHAN_REG_SSELT);
1226  writel(lidx, chanbase + D40_CHAN_REG_SDELT);
1227 
1228  /* Clear LNK which will be used by d40_chan_has_events() */
1229  writel(0, chanbase + D40_CHAN_REG_SSLNK);
1230  writel(0, chanbase + D40_CHAN_REG_SDLNK);
1231  }
1232 }
1233 
1234 static u32 d40_residue(struct d40_chan *d40c)
1235 {
1236  u32 num_elt;
1237 
1238  if (chan_is_logical(d40c))
1239  num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
1241  else {
1242  u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
1243  num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
1245  }
1246 
1247  return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
1248 }
1249 
1250 static bool d40_tx_is_linked(struct d40_chan *d40c)
1251 {
1252  bool is_link;
1253 
1254  if (chan_is_logical(d40c))
1255  is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
1256  else
1257  is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
1259 
1260  return is_link;
1261 }
1262 
1263 static int d40_pause(struct d40_chan *d40c)
1264 {
1265  int res = 0;
1266  unsigned long flags;
1267 
1268  if (!d40c->busy)
1269  return 0;
1270 
1271  pm_runtime_get_sync(d40c->base->dev);
1272  spin_lock_irqsave(&d40c->lock, flags);
1273 
1274  res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1275 
1276  pm_runtime_mark_last_busy(d40c->base->dev);
1277  pm_runtime_put_autosuspend(d40c->base->dev);
1278  spin_unlock_irqrestore(&d40c->lock, flags);
1279  return res;
1280 }
1281 
1282 static int d40_resume(struct d40_chan *d40c)
1283 {
1284  int res = 0;
1285  unsigned long flags;
1286 
1287  if (!d40c->busy)
1288  return 0;
1289 
1290  spin_lock_irqsave(&d40c->lock, flags);
1291  pm_runtime_get_sync(d40c->base->dev);
1292 
1293  /* If bytes left to transfer or linked tx resume job */
1294  if (d40_residue(d40c) || d40_tx_is_linked(d40c))
1295  res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1296 
1297  pm_runtime_mark_last_busy(d40c->base->dev);
1298  pm_runtime_put_autosuspend(d40c->base->dev);
1299  spin_unlock_irqrestore(&d40c->lock, flags);
1300  return res;
1301 }
1302 
1303 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1304 {
1305  struct d40_chan *d40c = container_of(tx->chan,
1306  struct d40_chan,
1307  chan);
1308  struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1309  unsigned long flags;
1311 
1312  spin_lock_irqsave(&d40c->lock, flags);
1313  cookie = dma_cookie_assign(tx);
1314  d40_desc_queue(d40c, d40d);
1315  spin_unlock_irqrestore(&d40c->lock, flags);
1316 
1317  return cookie;
1318 }
1319 
1320 static int d40_start(struct d40_chan *d40c)
1321 {
1322  return d40_channel_execute_command(d40c, D40_DMA_RUN);
1323 }
1324 
1325 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1326 {
1327  struct d40_desc *d40d;
1328  int err;
1329 
1330  /* Start queued jobs, if any */
1331  d40d = d40_first_queued(d40c);
1332 
1333  if (d40d != NULL) {
1334  if (!d40c->busy) {
1335  d40c->busy = true;
1336  pm_runtime_get_sync(d40c->base->dev);
1337  }
1338 
1339  /* Remove from queue */
1340  d40_desc_remove(d40d);
1341 
1342  /* Add to active queue */
1343  d40_desc_submit(d40c, d40d);
1344 
1345  /* Initiate DMA job */
1346  d40_desc_load(d40c, d40d);
1347 
1348  /* Start dma job */
1349  err = d40_start(d40c);
1350 
1351  if (err)
1352  return NULL;
1353  }
1354 
1355  return d40d;
1356 }
1357 
1358 /* called from interrupt context */
1359 static void dma_tc_handle(struct d40_chan *d40c)
1360 {
1361  struct d40_desc *d40d;
1362 
1363  /* Get first active entry from list */
1364  d40d = d40_first_active_get(d40c);
1365 
1366  if (d40d == NULL)
1367  return;
1368 
1369  if (d40d->cyclic) {
1370  /*
1371  * If this was a paritially loaded list, we need to reloaded
1372  * it, and only when the list is completed. We need to check
1373  * for done because the interrupt will hit for every link, and
1374  * not just the last one.
1375  */
1376  if (d40d->lli_current < d40d->lli_len
1377  && !d40_tx_is_linked(d40c)
1378  && !d40_residue(d40c)) {
1379  d40_lcla_free_all(d40c, d40d);
1380  d40_desc_load(d40c, d40d);
1381  (void) d40_start(d40c);
1382 
1383  if (d40d->lli_current == d40d->lli_len)
1384  d40d->lli_current = 0;
1385  }
1386  } else {
1387  d40_lcla_free_all(d40c, d40d);
1388 
1389  if (d40d->lli_current < d40d->lli_len) {
1390  d40_desc_load(d40c, d40d);
1391  /* Start dma job */
1392  (void) d40_start(d40c);
1393  return;
1394  }
1395 
1396  if (d40_queue_start(d40c) == NULL)
1397  d40c->busy = false;
1398  pm_runtime_mark_last_busy(d40c->base->dev);
1399  pm_runtime_put_autosuspend(d40c->base->dev);
1400  }
1401 
1402  d40c->pending_tx++;
1403  tasklet_schedule(&d40c->tasklet);
1404 
1405 }
1406 
1407 static void dma_tasklet(unsigned long data)
1408 {
1409  struct d40_chan *d40c = (struct d40_chan *) data;
1410  struct d40_desc *d40d;
1411  unsigned long flags;
1413  void *callback_param;
1414 
1415  spin_lock_irqsave(&d40c->lock, flags);
1416 
1417  /* Get first active entry from list */
1418  d40d = d40_first_active_get(d40c);
1419  if (d40d == NULL)
1420  goto err;
1421 
1422  if (!d40d->cyclic)
1423  dma_cookie_complete(&d40d->txd);
1424 
1425  /*
1426  * If terminating a channel pending_tx is set to zero.
1427  * This prevents any finished active jobs to return to the client.
1428  */
1429  if (d40c->pending_tx == 0) {
1430  spin_unlock_irqrestore(&d40c->lock, flags);
1431  return;
1432  }
1433 
1434  /* Callback to client */
1435  callback = d40d->txd.callback;
1436  callback_param = d40d->txd.callback_param;
1437 
1438  if (!d40d->cyclic) {
1439  if (async_tx_test_ack(&d40d->txd)) {
1440  d40_desc_remove(d40d);
1441  d40_desc_free(d40c, d40d);
1442  } else {
1443  if (!d40d->is_in_client_list) {
1444  d40_desc_remove(d40d);
1445  d40_lcla_free_all(d40c, d40d);
1446  list_add_tail(&d40d->node, &d40c->client);
1447  d40d->is_in_client_list = true;
1448  }
1449  }
1450  }
1451 
1452  d40c->pending_tx--;
1453 
1454  if (d40c->pending_tx)
1455  tasklet_schedule(&d40c->tasklet);
1456 
1457  spin_unlock_irqrestore(&d40c->lock, flags);
1458 
1459  if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1460  callback(callback_param);
1461 
1462  return;
1463 
1464 err:
1465  /* Rescue manouver if receiving double interrupts */
1466  if (d40c->pending_tx > 0)
1467  d40c->pending_tx--;
1468  spin_unlock_irqrestore(&d40c->lock, flags);
1469 }
1470 
1471 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1472 {
1473  static const struct d40_interrupt_lookup il[] = {
1474  {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
1475  {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1476  {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1477  {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1478  {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
1479  {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
1480  {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
1481  {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
1484  };
1485 
1486  int i;
1487  u32 regs[ARRAY_SIZE(il)];
1488  u32 idx;
1489  u32 row;
1490  long chan = -1;
1491  struct d40_chan *d40c;
1492  unsigned long flags;
1493  struct d40_base *base = data;
1494 
1495  spin_lock_irqsave(&base->interrupt_lock, flags);
1496 
1497  /* Read interrupt status of both logical and physical channels */
1498  for (i = 0; i < ARRAY_SIZE(il); i++)
1499  regs[i] = readl(base->virtbase + il[i].src);
1500 
1501  for (;;) {
1502 
1503  chan = find_next_bit((unsigned long *)regs,
1504  BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1505 
1506  /* No more set bits found? */
1507  if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1508  break;
1509 
1510  row = chan / BITS_PER_LONG;
1511  idx = chan & (BITS_PER_LONG - 1);
1512 
1513  /* ACK interrupt */
1514  writel(1 << idx, base->virtbase + il[row].clr);
1515 
1516  if (il[row].offset == D40_PHY_CHAN)
1517  d40c = base->lookup_phy_chans[idx];
1518  else
1519  d40c = base->lookup_log_chans[il[row].offset + idx];
1520  spin_lock(&d40c->lock);
1521 
1522  if (!il[row].is_error)
1523  dma_tc_handle(d40c);
1524  else
1525  d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1526  chan, il[row].offset, idx);
1527 
1528  spin_unlock(&d40c->lock);
1529  }
1530 
1531  spin_unlock_irqrestore(&base->interrupt_lock, flags);
1532 
1533  return IRQ_HANDLED;
1534 }
1535 
1536 static int d40_validate_conf(struct d40_chan *d40c,
1537  struct stedma40_chan_cfg *conf)
1538 {
1539  int res = 0;
1540  u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1541  u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1542  bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1543 
1544  if (!conf->dir) {
1545  chan_err(d40c, "Invalid direction.\n");
1546  res = -EINVAL;
1547  }
1548 
1549  if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1550  d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1551  d40c->runtime_addr == 0) {
1552 
1553  chan_err(d40c, "Invalid TX channel address (%d)\n",
1554  conf->dst_dev_type);
1555  res = -EINVAL;
1556  }
1557 
1558  if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1559  d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1560  d40c->runtime_addr == 0) {
1561  chan_err(d40c, "Invalid RX channel address (%d)\n",
1562  conf->src_dev_type);
1563  res = -EINVAL;
1564  }
1565 
1566  if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1567  dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1568  chan_err(d40c, "Invalid dst\n");
1569  res = -EINVAL;
1570  }
1571 
1572  if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1573  src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1574  chan_err(d40c, "Invalid src\n");
1575  res = -EINVAL;
1576  }
1577 
1578  if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1579  dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1580  chan_err(d40c, "No event line\n");
1581  res = -EINVAL;
1582  }
1583 
1584  if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1585  (src_event_group != dst_event_group)) {
1586  chan_err(d40c, "Invalid event group\n");
1587  res = -EINVAL;
1588  }
1589 
1590  if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1591  /*
1592  * DMAC HW supports it. Will be added to this driver,
1593  * in case any dma client requires it.
1594  */
1595  chan_err(d40c, "periph to periph not supported\n");
1596  res = -EINVAL;
1597  }
1598 
1599  if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1600  (1 << conf->src_info.data_width) !=
1601  d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1602  (1 << conf->dst_info.data_width)) {
1603  /*
1604  * The DMAC hardware only supports
1605  * src (burst x width) == dst (burst x width)
1606  */
1607 
1608  chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1609  res = -EINVAL;
1610  }
1611 
1612  return res;
1613 }
1614 
1615 static bool d40_alloc_mask_set(struct d40_phy_res *phy,
1616  bool is_src, int log_event_line, bool is_log,
1617  bool *first_user)
1618 {
1619  unsigned long flags;
1620  spin_lock_irqsave(&phy->lock, flags);
1621 
1622  *first_user = ((phy->allocated_src | phy->allocated_dst)
1623  == D40_ALLOC_FREE);
1624 
1625  if (!is_log) {
1626  /* Physical interrupts are masked per physical full channel */
1627  if (phy->allocated_src == D40_ALLOC_FREE &&
1628  phy->allocated_dst == D40_ALLOC_FREE) {
1631  goto found;
1632  } else
1633  goto not_found;
1634  }
1635 
1636  /* Logical channel */
1637  if (is_src) {
1638  if (phy->allocated_src == D40_ALLOC_PHY)
1639  goto not_found;
1640 
1641  if (phy->allocated_src == D40_ALLOC_FREE)
1643 
1644  if (!(phy->allocated_src & (1 << log_event_line))) {
1645  phy->allocated_src |= 1 << log_event_line;
1646  goto found;
1647  } else
1648  goto not_found;
1649  } else {
1650  if (phy->allocated_dst == D40_ALLOC_PHY)
1651  goto not_found;
1652 
1653  if (phy->allocated_dst == D40_ALLOC_FREE)
1655 
1656  if (!(phy->allocated_dst & (1 << log_event_line))) {
1657  phy->allocated_dst |= 1 << log_event_line;
1658  goto found;
1659  } else
1660  goto not_found;
1661  }
1662 
1663 not_found:
1664  spin_unlock_irqrestore(&phy->lock, flags);
1665  return false;
1666 found:
1667  spin_unlock_irqrestore(&phy->lock, flags);
1668  return true;
1669 }
1670 
1671 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1672  int log_event_line)
1673 {
1674  unsigned long flags;
1675  bool is_free = false;
1676 
1677  spin_lock_irqsave(&phy->lock, flags);
1678  if (!log_event_line) {
1681  is_free = true;
1682  goto out;
1683  }
1684 
1685  /* Logical channel */
1686  if (is_src) {
1687  phy->allocated_src &= ~(1 << log_event_line);
1688  if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1690  } else {
1691  phy->allocated_dst &= ~(1 << log_event_line);
1692  if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1694  }
1695 
1696  is_free = ((phy->allocated_src | phy->allocated_dst) ==
1697  D40_ALLOC_FREE);
1698 
1699 out:
1700  spin_unlock_irqrestore(&phy->lock, flags);
1701 
1702  return is_free;
1703 }
1704 
1705 static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
1706 {
1707  int dev_type;
1708  int event_group;
1709  int event_line;
1710  struct d40_phy_res *phys;
1711  int i;
1712  int j;
1713  int log_num;
1714  bool is_src;
1715  bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1716 
1717  phys = d40c->base->phy_res;
1718 
1719  if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1720  dev_type = d40c->dma_cfg.src_dev_type;
1721  log_num = 2 * dev_type;
1722  is_src = true;
1723  } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1724  d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1725  /* dst event lines are used for logical memcpy */
1726  dev_type = d40c->dma_cfg.dst_dev_type;
1727  log_num = 2 * dev_type + 1;
1728  is_src = false;
1729  } else
1730  return -EINVAL;
1731 
1732  event_group = D40_TYPE_TO_GROUP(dev_type);
1733  event_line = D40_TYPE_TO_EVENT(dev_type);
1734 
1735  if (!is_log) {
1736  if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1737  /* Find physical half channel */
1738  for (i = 0; i < d40c->base->num_phy_chans; i++) {
1739 
1740  if (d40_alloc_mask_set(&phys[i], is_src,
1741  0, is_log,
1742  first_phy_user))
1743  goto found_phy;
1744  }
1745  } else
1746  for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1747  int phy_num = j + event_group * 2;
1748  for (i = phy_num; i < phy_num + 2; i++) {
1749  if (d40_alloc_mask_set(&phys[i],
1750  is_src,
1751  0,
1752  is_log,
1753  first_phy_user))
1754  goto found_phy;
1755  }
1756  }
1757  return -EINVAL;
1758 found_phy:
1759  d40c->phy_chan = &phys[i];
1760  d40c->log_num = D40_PHY_CHAN;
1761  goto out;
1762  }
1763  if (dev_type == -1)
1764  return -EINVAL;
1765 
1766  /* Find logical channel */
1767  for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1768  int phy_num = j + event_group * 2;
1769 
1770  if (d40c->dma_cfg.use_fixed_channel) {
1771  i = d40c->dma_cfg.phy_channel;
1772 
1773  if ((i != phy_num) && (i != phy_num + 1)) {
1774  dev_err(chan2dev(d40c),
1775  "invalid fixed phy channel %d\n", i);
1776  return -EINVAL;
1777  }
1778 
1779  if (d40_alloc_mask_set(&phys[i], is_src, event_line,
1780  is_log, first_phy_user))
1781  goto found_log;
1782 
1783  dev_err(chan2dev(d40c),
1784  "could not allocate fixed phy channel %d\n", i);
1785  return -EINVAL;
1786  }
1787 
1788  /*
1789  * Spread logical channels across all available physical rather
1790  * than pack every logical channel at the first available phy
1791  * channels.
1792  */
1793  if (is_src) {
1794  for (i = phy_num; i < phy_num + 2; i++) {
1795  if (d40_alloc_mask_set(&phys[i], is_src,
1796  event_line, is_log,
1797  first_phy_user))
1798  goto found_log;
1799  }
1800  } else {
1801  for (i = phy_num + 1; i >= phy_num; i--) {
1802  if (d40_alloc_mask_set(&phys[i], is_src,
1803  event_line, is_log,
1804  first_phy_user))
1805  goto found_log;
1806  }
1807  }
1808  }
1809  return -EINVAL;
1810 
1811 found_log:
1812  d40c->phy_chan = &phys[i];
1813  d40c->log_num = log_num;
1814 out:
1815 
1816  if (is_log)
1817  d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1818  else
1819  d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1820 
1821  return 0;
1822 
1823 }
1824 
1825 static int d40_config_memcpy(struct d40_chan *d40c)
1826 {
1827  dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1828 
1829  if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1830  d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1831  d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1832  d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1833  memcpy[d40c->chan.chan_id];
1834 
1835  } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1836  dma_has_cap(DMA_SLAVE, cap)) {
1837  d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1838  } else {
1839  chan_err(d40c, "No memcpy\n");
1840  return -EINVAL;
1841  }
1842 
1843  return 0;
1844 }
1845 
1846 static int d40_free_dma(struct d40_chan *d40c)
1847 {
1848 
1849  int res = 0;
1850  u32 event;
1851  struct d40_phy_res *phy = d40c->phy_chan;
1852  bool is_src;
1853 
1854  /* Terminate all queued and active transfers */
1855  d40_term_all(d40c);
1856 
1857  if (phy == NULL) {
1858  chan_err(d40c, "phy == null\n");
1859  return -EINVAL;
1860  }
1861 
1862  if (phy->allocated_src == D40_ALLOC_FREE &&
1863  phy->allocated_dst == D40_ALLOC_FREE) {
1864  chan_err(d40c, "channel already free\n");
1865  return -EINVAL;
1866  }
1867 
1868  if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1869  d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1870  event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1871  is_src = false;
1872  } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1873  event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1874  is_src = true;
1875  } else {
1876  chan_err(d40c, "Unknown direction\n");
1877  return -EINVAL;
1878  }
1879 
1880  pm_runtime_get_sync(d40c->base->dev);
1881  res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1882  if (res) {
1883  chan_err(d40c, "stop failed\n");
1884  goto out;
1885  }
1886 
1887  d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0);
1888 
1889  if (chan_is_logical(d40c))
1890  d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1891  else
1892  d40c->base->lookup_phy_chans[phy->num] = NULL;
1893 
1894  if (d40c->busy) {
1895  pm_runtime_mark_last_busy(d40c->base->dev);
1896  pm_runtime_put_autosuspend(d40c->base->dev);
1897  }
1898 
1899  d40c->busy = false;
1900  d40c->phy_chan = NULL;
1901  d40c->configured = false;
1902 out:
1903 
1904  pm_runtime_mark_last_busy(d40c->base->dev);
1905  pm_runtime_put_autosuspend(d40c->base->dev);
1906  return res;
1907 }
1908 
1909 static bool d40_is_paused(struct d40_chan *d40c)
1910 {
1911  void __iomem *chanbase = chan_base(d40c);
1912  bool is_paused = false;
1913  unsigned long flags;
1914  void __iomem *active_reg;
1915  u32 status;
1916  u32 event;
1917 
1918  spin_lock_irqsave(&d40c->lock, flags);
1919 
1920  if (chan_is_physical(d40c)) {
1921  if (d40c->phy_chan->num % 2 == 0)
1922  active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1923  else
1924  active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1925 
1926  status = (readl(active_reg) &
1927  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1928  D40_CHAN_POS(d40c->phy_chan->num);
1929  if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1930  is_paused = true;
1931 
1932  goto _exit;
1933  }
1934 
1935  if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1936  d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1937  event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1938  status = readl(chanbase + D40_CHAN_REG_SDLNK);
1939  } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1940  event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1941  status = readl(chanbase + D40_CHAN_REG_SSLNK);
1942  } else {
1943  chan_err(d40c, "Unknown direction\n");
1944  goto _exit;
1945  }
1946 
1947  status = (status & D40_EVENTLINE_MASK(event)) >>
1948  D40_EVENTLINE_POS(event);
1949 
1950  if (status != D40_DMA_RUN)
1951  is_paused = true;
1952 _exit:
1953  spin_unlock_irqrestore(&d40c->lock, flags);
1954  return is_paused;
1955 
1956 }
1957 
1958 
1959 static u32 stedma40_residue(struct dma_chan *chan)
1960 {
1961  struct d40_chan *d40c =
1962  container_of(chan, struct d40_chan, chan);
1963  u32 bytes_left;
1964  unsigned long flags;
1965 
1966  spin_lock_irqsave(&d40c->lock, flags);
1967  bytes_left = d40_residue(d40c);
1968  spin_unlock_irqrestore(&d40c->lock, flags);
1969 
1970  return bytes_left;
1971 }
1972 
1973 static int
1974 d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
1975  struct scatterlist *sg_src, struct scatterlist *sg_dst,
1976  unsigned int sg_len, dma_addr_t src_dev_addr,
1977  dma_addr_t dst_dev_addr)
1978 {
1979  struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1980  struct stedma40_half_channel_info *src_info = &cfg->src_info;
1981  struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
1982  int ret;
1983 
1984  ret = d40_log_sg_to_lli(sg_src, sg_len,
1985  src_dev_addr,
1986  desc->lli_log.src,
1987  chan->log_def.lcsp1,
1988  src_info->data_width,
1989  dst_info->data_width);
1990 
1991  ret = d40_log_sg_to_lli(sg_dst, sg_len,
1992  dst_dev_addr,
1993  desc->lli_log.dst,
1994  chan->log_def.lcsp3,
1995  dst_info->data_width,
1996  src_info->data_width);
1997 
1998  return ret < 0 ? ret : 0;
1999 }
2000 
2001 static int
2002 d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
2003  struct scatterlist *sg_src, struct scatterlist *sg_dst,
2004  unsigned int sg_len, dma_addr_t src_dev_addr,
2005  dma_addr_t dst_dev_addr)
2006 {
2007  struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2008  struct stedma40_half_channel_info *src_info = &cfg->src_info;
2009  struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
2010  unsigned long flags = 0;
2011  int ret;
2012 
2013  if (desc->cyclic)
2014  flags |= LLI_CYCLIC | LLI_TERM_INT;
2015 
2016  ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
2017  desc->lli_phy.src,
2018  virt_to_phys(desc->lli_phy.src),
2019  chan->src_def_cfg,
2020  src_info, dst_info, flags);
2021 
2022  ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
2023  desc->lli_phy.dst,
2024  virt_to_phys(desc->lli_phy.dst),
2025  chan->dst_def_cfg,
2026  dst_info, src_info, flags);
2027 
2028  dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
2029  desc->lli_pool.size, DMA_TO_DEVICE);
2030 
2031  return ret < 0 ? ret : 0;
2032 }
2033 
2034 
2035 static struct d40_desc *
2036 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
2037  unsigned int sg_len, unsigned long dma_flags)
2038 {
2039  struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2040  struct d40_desc *desc;
2041  int ret;
2042 
2043  desc = d40_desc_get(chan);
2044  if (!desc)
2045  return NULL;
2046 
2047  desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
2048  cfg->dst_info.data_width);
2049  if (desc->lli_len < 0) {
2050  chan_err(chan, "Unaligned size\n");
2051  goto err;
2052  }
2053 
2054  ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
2055  if (ret < 0) {
2056  chan_err(chan, "Could not allocate lli\n");
2057  goto err;
2058  }
2059 
2060 
2061  desc->lli_current = 0;
2062  desc->txd.flags = dma_flags;
2063  desc->txd.tx_submit = d40_tx_submit;
2064 
2065  dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
2066 
2067  return desc;
2068 
2069 err:
2070  d40_desc_free(chan, desc);
2071  return NULL;
2072 }
2073 
2074 static dma_addr_t
2075 d40_get_dev_addr(struct d40_chan *chan, enum dma_transfer_direction direction)
2076 {
2077  struct stedma40_platform_data *plat = chan->base->plat_data;
2078  struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2079  dma_addr_t addr = 0;
2080 
2081  if (chan->runtime_addr)
2082  return chan->runtime_addr;
2083 
2084  if (direction == DMA_DEV_TO_MEM)
2085  addr = plat->dev_rx[cfg->src_dev_type];
2086  else if (direction == DMA_MEM_TO_DEV)
2087  addr = plat->dev_tx[cfg->dst_dev_type];
2088 
2089  return addr;
2090 }
2091 
2092 static struct dma_async_tx_descriptor *
2093 d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
2094  struct scatterlist *sg_dst, unsigned int sg_len,
2095  enum dma_transfer_direction direction, unsigned long dma_flags)
2096 {
2097  struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
2098  dma_addr_t src_dev_addr = 0;
2099  dma_addr_t dst_dev_addr = 0;
2100  struct d40_desc *desc;
2101  unsigned long flags;
2102  int ret;
2103 
2104  if (!chan->phy_chan) {
2105  chan_err(chan, "Cannot prepare unallocated channel\n");
2106  return NULL;
2107  }
2108 
2109 
2110  spin_lock_irqsave(&chan->lock, flags);
2111 
2112  desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
2113  if (desc == NULL)
2114  goto err;
2115 
2116  if (sg_next(&sg_src[sg_len - 1]) == sg_src)
2117  desc->cyclic = true;
2118 
2119  if (direction != DMA_TRANS_NONE) {
2120  dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
2121 
2122  if (direction == DMA_DEV_TO_MEM)
2123  src_dev_addr = dev_addr;
2124  else if (direction == DMA_MEM_TO_DEV)
2125  dst_dev_addr = dev_addr;
2126  }
2127 
2128  if (chan_is_logical(chan))
2129  ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
2130  sg_len, src_dev_addr, dst_dev_addr);
2131  else
2132  ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
2133  sg_len, src_dev_addr, dst_dev_addr);
2134 
2135  if (ret) {
2136  chan_err(chan, "Failed to prepare %s sg job: %d\n",
2137  chan_is_logical(chan) ? "log" : "phy", ret);
2138  goto err;
2139  }
2140 
2141  /*
2142  * add descriptor to the prepare queue in order to be able
2143  * to free them later in terminate_all
2144  */
2145  list_add_tail(&desc->node, &chan->prepare_queue);
2146 
2147  spin_unlock_irqrestore(&chan->lock, flags);
2148 
2149  return &desc->txd;
2150 
2151 err:
2152  if (desc)
2153  d40_desc_free(chan, desc);
2154  spin_unlock_irqrestore(&chan->lock, flags);
2155  return NULL;
2156 }
2157 
2158 bool stedma40_filter(struct dma_chan *chan, void *data)
2159 {
2160  struct stedma40_chan_cfg *info = data;
2161  struct d40_chan *d40c =
2162  container_of(chan, struct d40_chan, chan);
2163  int err;
2164 
2165  if (data) {
2166  err = d40_validate_conf(d40c, info);
2167  if (!err)
2168  d40c->dma_cfg = *info;
2169  } else
2170  err = d40_config_memcpy(d40c);
2171 
2172  if (!err)
2173  d40c->configured = true;
2174 
2175  return err == 0;
2176 }
2178 
2179 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
2180 {
2181  bool realtime = d40c->dma_cfg.realtime;
2182  bool highprio = d40c->dma_cfg.high_priority;
2183  u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
2184  u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
2185  u32 event = D40_TYPE_TO_EVENT(dev_type);
2186  u32 group = D40_TYPE_TO_GROUP(dev_type);
2187  u32 bit = 1 << event;
2188 
2189  /* Destination event lines are stored in the upper halfword */
2190  if (!src)
2191  bit <<= 16;
2192 
2193  writel(bit, d40c->base->virtbase + prioreg + group * 4);
2194  writel(bit, d40c->base->virtbase + rtreg + group * 4);
2195 }
2196 
2197 static void d40_set_prio_realtime(struct d40_chan *d40c)
2198 {
2199  if (d40c->base->rev < 3)
2200  return;
2201 
2202  if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
2203  (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
2204  __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
2205 
2206  if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
2207  (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
2208  __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
2209 }
2210 
2211 /* DMA ENGINE functions */
2212 static int d40_alloc_chan_resources(struct dma_chan *chan)
2213 {
2214  int err;
2215  unsigned long flags;
2216  struct d40_chan *d40c =
2217  container_of(chan, struct d40_chan, chan);
2218  bool is_free_phy;
2219  spin_lock_irqsave(&d40c->lock, flags);
2220 
2221  dma_cookie_init(chan);
2222 
2223  /* If no dma configuration is set use default configuration (memcpy) */
2224  if (!d40c->configured) {
2225  err = d40_config_memcpy(d40c);
2226  if (err) {
2227  chan_err(d40c, "Failed to configure memcpy channel\n");
2228  goto fail;
2229  }
2230  }
2231 
2232  err = d40_allocate_channel(d40c, &is_free_phy);
2233  if (err) {
2234  chan_err(d40c, "Failed to allocate channel\n");
2235  d40c->configured = false;
2236  goto fail;
2237  }
2238 
2239  pm_runtime_get_sync(d40c->base->dev);
2240  /* Fill in basic CFG register values */
2241  d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
2242  &d40c->dst_def_cfg, chan_is_logical(d40c));
2243 
2244  d40_set_prio_realtime(d40c);
2245 
2246  if (chan_is_logical(d40c)) {
2247  d40_log_cfg(&d40c->dma_cfg,
2248  &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2249 
2250  if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
2251  d40c->lcpa = d40c->base->lcpa_base +
2252  d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
2253  else
2254  d40c->lcpa = d40c->base->lcpa_base +
2255  d40c->dma_cfg.dst_dev_type *
2257  }
2258 
2259  dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",
2260  chan_is_logical(d40c) ? "logical" : "physical",
2261  d40c->phy_chan->num,
2262  d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");
2263 
2264 
2265  /*
2266  * Only write channel configuration to the DMA if the physical
2267  * resource is free. In case of multiple logical channels
2268  * on the same physical resource, only the first write is necessary.
2269  */
2270  if (is_free_phy)
2271  d40_config_write(d40c);
2272 fail:
2273  pm_runtime_mark_last_busy(d40c->base->dev);
2274  pm_runtime_put_autosuspend(d40c->base->dev);
2275  spin_unlock_irqrestore(&d40c->lock, flags);
2276  return err;
2277 }
2278 
2279 static void d40_free_chan_resources(struct dma_chan *chan)
2280 {
2281  struct d40_chan *d40c =
2282  container_of(chan, struct d40_chan, chan);
2283  int err;
2284  unsigned long flags;
2285 
2286  if (d40c->phy_chan == NULL) {
2287  chan_err(d40c, "Cannot free unallocated channel\n");
2288  return;
2289  }
2290 
2291 
2292  spin_lock_irqsave(&d40c->lock, flags);
2293 
2294  err = d40_free_dma(d40c);
2295 
2296  if (err)
2297  chan_err(d40c, "Failed to free channel\n");
2298  spin_unlock_irqrestore(&d40c->lock, flags);
2299 }
2300 
2301 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
2302  dma_addr_t dst,
2303  dma_addr_t src,
2304  size_t size,
2305  unsigned long dma_flags)
2306 {
2307  struct scatterlist dst_sg;
2308  struct scatterlist src_sg;
2309 
2310  sg_init_table(&dst_sg, 1);
2311  sg_init_table(&src_sg, 1);
2312 
2313  sg_dma_address(&dst_sg) = dst;
2314  sg_dma_address(&src_sg) = src;
2315 
2316  sg_dma_len(&dst_sg) = size;
2317  sg_dma_len(&src_sg) = size;
2318 
2319  return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
2320 }
2321 
2322 static struct dma_async_tx_descriptor *
2323 d40_prep_memcpy_sg(struct dma_chan *chan,
2324  struct scatterlist *dst_sg, unsigned int dst_nents,
2325  struct scatterlist *src_sg, unsigned int src_nents,
2326  unsigned long dma_flags)
2327 {
2328  if (dst_nents != src_nents)
2329  return NULL;
2330 
2331  return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
2332 }
2333 
2334 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2335  struct scatterlist *sgl,
2336  unsigned int sg_len,
2337  enum dma_transfer_direction direction,
2338  unsigned long dma_flags,
2339  void *context)
2340 {
2341  if (direction != DMA_DEV_TO_MEM && direction != DMA_MEM_TO_DEV)
2342  return NULL;
2343 
2344  return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
2345 }
2346 
2347 static struct dma_async_tx_descriptor *
2348 dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2349  size_t buf_len, size_t period_len,
2350  enum dma_transfer_direction direction, unsigned long flags,
2351  void *context)
2352 {
2353  unsigned int periods = buf_len / period_len;
2354  struct dma_async_tx_descriptor *txd;
2355  struct scatterlist *sg;
2356  int i;
2357 
2358  sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT);
2359  for (i = 0; i < periods; i++) {
2360  sg_dma_address(&sg[i]) = dma_addr;
2361  sg_dma_len(&sg[i]) = period_len;
2362  dma_addr += period_len;
2363  }
2364 
2365  sg[periods].offset = 0;
2366  sg_dma_len(&sg[periods]) = 0;
2367  sg[periods].page_link =
2368  ((unsigned long)sg | 0x01) & ~0x02;
2369 
2370  txd = d40_prep_sg(chan, sg, sg, periods, direction,
2372 
2373  kfree(sg);
2374 
2375  return txd;
2376 }
2377 
2378 static enum dma_status d40_tx_status(struct dma_chan *chan,
2380  struct dma_tx_state *txstate)
2381 {
2382  struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2383  enum dma_status ret;
2384 
2385  if (d40c->phy_chan == NULL) {
2386  chan_err(d40c, "Cannot read status of unallocated channel\n");
2387  return -EINVAL;
2388  }
2389 
2390  ret = dma_cookie_status(chan, cookie, txstate);
2391  if (ret != DMA_SUCCESS)
2392  dma_set_residue(txstate, stedma40_residue(chan));
2393 
2394  if (d40_is_paused(d40c))
2395  ret = DMA_PAUSED;
2396 
2397  return ret;
2398 }
2399 
2400 static void d40_issue_pending(struct dma_chan *chan)
2401 {
2402  struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2403  unsigned long flags;
2404 
2405  if (d40c->phy_chan == NULL) {
2406  chan_err(d40c, "Channel is not allocated!\n");
2407  return;
2408  }
2409 
2410  spin_lock_irqsave(&d40c->lock, flags);
2411 
2412  list_splice_tail_init(&d40c->pending_queue, &d40c->queue);
2413 
2414  /* Busy means that queued jobs are already being processed */
2415  if (!d40c->busy)
2416  (void) d40_queue_start(d40c);
2417 
2418  spin_unlock_irqrestore(&d40c->lock, flags);
2419 }
2420 
2421 static void d40_terminate_all(struct dma_chan *chan)
2422 {
2423  unsigned long flags;
2424  struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2425  int ret;
2426 
2427  spin_lock_irqsave(&d40c->lock, flags);
2428 
2429  pm_runtime_get_sync(d40c->base->dev);
2430  ret = d40_channel_execute_command(d40c, D40_DMA_STOP);
2431  if (ret)
2432  chan_err(d40c, "Failed to stop channel\n");
2433 
2434  d40_term_all(d40c);
2435  pm_runtime_mark_last_busy(d40c->base->dev);
2436  pm_runtime_put_autosuspend(d40c->base->dev);
2437  if (d40c->busy) {
2438  pm_runtime_mark_last_busy(d40c->base->dev);
2439  pm_runtime_put_autosuspend(d40c->base->dev);
2440  }
2441  d40c->busy = false;
2442 
2443  spin_unlock_irqrestore(&d40c->lock, flags);
2444 }
2445 
2446 static int
2447 dma40_config_to_halfchannel(struct d40_chan *d40c,
2450  u32 maxburst)
2451 {
2452  enum stedma40_periph_data_width addr_width;
2453  int psize;
2454 
2455  switch (width) {
2457  addr_width = STEDMA40_BYTE_WIDTH;
2458  break;
2460  addr_width = STEDMA40_HALFWORD_WIDTH;
2461  break;
2463  addr_width = STEDMA40_WORD_WIDTH;
2464  break;
2466  addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2467  break;
2468  default:
2469  dev_err(d40c->base->dev,
2470  "illegal peripheral address width "
2471  "requested (%d)\n",
2472  width);
2473  return -EINVAL;
2474  }
2475 
2476  if (chan_is_logical(d40c)) {
2477  if (maxburst >= 16)
2478  psize = STEDMA40_PSIZE_LOG_16;
2479  else if (maxburst >= 8)
2480  psize = STEDMA40_PSIZE_LOG_8;
2481  else if (maxburst >= 4)
2482  psize = STEDMA40_PSIZE_LOG_4;
2483  else
2484  psize = STEDMA40_PSIZE_LOG_1;
2485  } else {
2486  if (maxburst >= 16)
2487  psize = STEDMA40_PSIZE_PHY_16;
2488  else if (maxburst >= 8)
2489  psize = STEDMA40_PSIZE_PHY_8;
2490  else if (maxburst >= 4)
2491  psize = STEDMA40_PSIZE_PHY_4;
2492  else
2493  psize = STEDMA40_PSIZE_PHY_1;
2494  }
2495 
2496  info->data_width = addr_width;
2497  info->psize = psize;
2499 
2500  return 0;
2501 }
2502 
2503 /* Runtime reconfiguration extension */
2504 static int d40_set_runtime_config(struct dma_chan *chan,
2505  struct dma_slave_config *config)
2506 {
2507  struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2508  struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2509  enum dma_slave_buswidth src_addr_width, dst_addr_width;
2510  dma_addr_t config_addr;
2511  u32 src_maxburst, dst_maxburst;
2512  int ret;
2513 
2514  src_addr_width = config->src_addr_width;
2515  src_maxburst = config->src_maxburst;
2516  dst_addr_width = config->dst_addr_width;
2517  dst_maxburst = config->dst_maxburst;
2518 
2519  if (config->direction == DMA_DEV_TO_MEM) {
2520  dma_addr_t dev_addr_rx =
2521  d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2522 
2523  config_addr = config->src_addr;
2524  if (dev_addr_rx)
2525  dev_dbg(d40c->base->dev,
2526  "channel has a pre-wired RX address %08x "
2527  "overriding with %08x\n",
2528  dev_addr_rx, config_addr);
2529  if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2530  dev_dbg(d40c->base->dev,
2531  "channel was not configured for peripheral "
2532  "to memory transfer (%d) overriding\n",
2533  cfg->dir);
2534  cfg->dir = STEDMA40_PERIPH_TO_MEM;
2535 
2536  /* Configure the memory side */
2537  if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2538  dst_addr_width = src_addr_width;
2539  if (dst_maxburst == 0)
2540  dst_maxburst = src_maxburst;
2541 
2542  } else if (config->direction == DMA_MEM_TO_DEV) {
2543  dma_addr_t dev_addr_tx =
2544  d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2545 
2546  config_addr = config->dst_addr;
2547  if (dev_addr_tx)
2548  dev_dbg(d40c->base->dev,
2549  "channel has a pre-wired TX address %08x "
2550  "overriding with %08x\n",
2551  dev_addr_tx, config_addr);
2552  if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2553  dev_dbg(d40c->base->dev,
2554  "channel was not configured for memory "
2555  "to peripheral transfer (%d) overriding\n",
2556  cfg->dir);
2557  cfg->dir = STEDMA40_MEM_TO_PERIPH;
2558 
2559  /* Configure the memory side */
2560  if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2561  src_addr_width = dst_addr_width;
2562  if (src_maxburst == 0)
2563  src_maxburst = dst_maxburst;
2564  } else {
2565  dev_err(d40c->base->dev,
2566  "unrecognized channel direction %d\n",
2567  config->direction);
2568  return -EINVAL;
2569  }
2570 
2571  if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {
2572  dev_err(d40c->base->dev,
2573  "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
2574  src_maxburst,
2575  src_addr_width,
2576  dst_maxburst,
2577  dst_addr_width);
2578  return -EINVAL;
2579  }
2580 
2581  ret = dma40_config_to_halfchannel(d40c, &cfg->src_info,
2582  src_addr_width,
2583  src_maxburst);
2584  if (ret)
2585  return ret;
2586 
2587  ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info,
2588  dst_addr_width,
2589  dst_maxburst);
2590  if (ret)
2591  return ret;
2592 
2593  /* Fill in register values */
2594  if (chan_is_logical(d40c))
2595  d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2596  else
2597  d40_phy_cfg(cfg, &d40c->src_def_cfg,
2598  &d40c->dst_def_cfg, false);
2599 
2600  /* These settings will take precedence later */
2601  d40c->runtime_addr = config_addr;
2602  d40c->runtime_direction = config->direction;
2603  dev_dbg(d40c->base->dev,
2604  "configured channel %s for %s, data width %d/%d, "
2605  "maxburst %d/%d elements, LE, no flow control\n",
2606  dma_chan_name(chan),
2607  (config->direction == DMA_DEV_TO_MEM) ? "RX" : "TX",
2608  src_addr_width, dst_addr_width,
2609  src_maxburst, dst_maxburst);
2610 
2611  return 0;
2612 }
2613 
2614 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2615  unsigned long arg)
2616 {
2617  struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2618 
2619  if (d40c->phy_chan == NULL) {
2620  chan_err(d40c, "Channel is not allocated!\n");
2621  return -EINVAL;
2622  }
2623 
2624  switch (cmd) {
2625  case DMA_TERMINATE_ALL:
2626  d40_terminate_all(chan);
2627  return 0;
2628  case DMA_PAUSE:
2629  return d40_pause(d40c);
2630  case DMA_RESUME:
2631  return d40_resume(d40c);
2632  case DMA_SLAVE_CONFIG:
2633  return d40_set_runtime_config(chan,
2634  (struct dma_slave_config *) arg);
2635  default:
2636  break;
2637  }
2638 
2639  /* Other commands are unimplemented */
2640  return -ENXIO;
2641 }
2642 
2643 /* Initialization functions */
2644 
2645 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2646  struct d40_chan *chans, int offset,
2647  int num_chans)
2648 {
2649  int i = 0;
2650  struct d40_chan *d40c;
2651 
2652  INIT_LIST_HEAD(&dma->channels);
2653 
2654  for (i = offset; i < offset + num_chans; i++) {
2655  d40c = &chans[i];
2656  d40c->base = base;
2657  d40c->chan.device = dma;
2658 
2659  spin_lock_init(&d40c->lock);
2660 
2661  d40c->log_num = D40_PHY_CHAN;
2662 
2663  INIT_LIST_HEAD(&d40c->active);
2664  INIT_LIST_HEAD(&d40c->queue);
2665  INIT_LIST_HEAD(&d40c->pending_queue);
2666  INIT_LIST_HEAD(&d40c->client);
2667  INIT_LIST_HEAD(&d40c->prepare_queue);
2668 
2669  tasklet_init(&d40c->tasklet, dma_tasklet,
2670  (unsigned long) d40c);
2671 
2672  list_add_tail(&d40c->chan.device_node,
2673  &dma->channels);
2674  }
2675 }
2676 
2677 static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2678 {
2679  if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
2680  dev->device_prep_slave_sg = d40_prep_slave_sg;
2681 
2682  if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2683  dev->device_prep_dma_memcpy = d40_prep_memcpy;
2684 
2685  /*
2686  * This controller can only access address at even
2687  * 32bit boundaries, i.e. 2^2
2688  */
2689  dev->copy_align = 2;
2690  }
2691 
2692  if (dma_has_cap(DMA_SG, dev->cap_mask))
2693  dev->device_prep_dma_sg = d40_prep_memcpy_sg;
2694 
2695  if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2696  dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2697 
2698  dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2699  dev->device_free_chan_resources = d40_free_chan_resources;
2700  dev->device_issue_pending = d40_issue_pending;
2701  dev->device_tx_status = d40_tx_status;
2702  dev->device_control = d40_control;
2703  dev->dev = base->dev;
2704 }
2705 
2706 static int __init d40_dmaengine_init(struct d40_base *base,
2707  int num_reserved_chans)
2708 {
2709  int err ;
2710 
2711  d40_chan_init(base, &base->dma_slave, base->log_chans,
2712  0, base->num_log_chans);
2713 
2714  dma_cap_zero(base->dma_slave.cap_mask);
2715  dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2716  dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2717 
2718  d40_ops_init(base, &base->dma_slave);
2719 
2720  err = dma_async_device_register(&base->dma_slave);
2721 
2722  if (err) {
2723  d40_err(base->dev, "Failed to register slave channels\n");
2724  goto failure1;
2725  }
2726 
2727  d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2728  base->num_log_chans, base->plat_data->memcpy_len);
2729 
2730  dma_cap_zero(base->dma_memcpy.cap_mask);
2731  dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2732  dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
2733 
2734  d40_ops_init(base, &base->dma_memcpy);
2735 
2736  err = dma_async_device_register(&base->dma_memcpy);
2737 
2738  if (err) {
2739  d40_err(base->dev,
2740  "Failed to regsiter memcpy only channels\n");
2741  goto failure2;
2742  }
2743 
2744  d40_chan_init(base, &base->dma_both, base->phy_chans,
2745  0, num_reserved_chans);
2746 
2747  dma_cap_zero(base->dma_both.cap_mask);
2748  dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2749  dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2750  dma_cap_set(DMA_SG, base->dma_both.cap_mask);
2751  dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2752 
2753  d40_ops_init(base, &base->dma_both);
2754  err = dma_async_device_register(&base->dma_both);
2755 
2756  if (err) {
2757  d40_err(base->dev,
2758  "Failed to register logical and physical capable channels\n");
2759  goto failure3;
2760  }
2761  return 0;
2762 failure3:
2764 failure2:
2766 failure1:
2767  return err;
2768 }
2769 
2770 /* Suspend resume functionality */
2771 #ifdef CONFIG_PM
2772 static int dma40_pm_suspend(struct device *dev)
2773 {
2774  struct platform_device *pdev = to_platform_device(dev);
2775  struct d40_base *base = platform_get_drvdata(pdev);
2776  int ret = 0;
2777  if (!pm_runtime_suspended(dev))
2778  return -EBUSY;
2779 
2780  if (base->lcpa_regulator)
2781  ret = regulator_disable(base->lcpa_regulator);
2782  return ret;
2783 }
2784 
2785 static int dma40_runtime_suspend(struct device *dev)
2786 {
2787  struct platform_device *pdev = to_platform_device(dev);
2788  struct d40_base *base = platform_get_drvdata(pdev);
2789 
2790  d40_save_restore_registers(base, true);
2791 
2792  /* Don't disable/enable clocks for v1 due to HW bugs */
2793  if (base->rev != 1)
2794  writel_relaxed(base->gcc_pwr_off_mask,
2795  base->virtbase + D40_DREG_GCC);
2796 
2797  return 0;
2798 }
2799 
2800 static int dma40_runtime_resume(struct device *dev)
2801 {
2802  struct platform_device *pdev = to_platform_device(dev);
2803  struct d40_base *base = platform_get_drvdata(pdev);
2804 
2805  if (base->initialized)
2806  d40_save_restore_registers(base, false);
2807 
2808  writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
2809  base->virtbase + D40_DREG_GCC);
2810  return 0;
2811 }
2812 
2813 static int dma40_resume(struct device *dev)
2814 {
2815  struct platform_device *pdev = to_platform_device(dev);
2816  struct d40_base *base = platform_get_drvdata(pdev);
2817  int ret = 0;
2818 
2819  if (base->lcpa_regulator)
2820  ret = regulator_enable(base->lcpa_regulator);
2821 
2822  return ret;
2823 }
2824 
2825 static const struct dev_pm_ops dma40_pm_ops = {
2826  .suspend = dma40_pm_suspend,
2827  .runtime_suspend = dma40_runtime_suspend,
2828  .runtime_resume = dma40_runtime_resume,
2829  .resume = dma40_resume,
2830 };
2831 #define DMA40_PM_OPS (&dma40_pm_ops)
2832 #else
2833 #define DMA40_PM_OPS NULL
2834 #endif
2835 
2836 /* Initialization functions. */
2837 
2838 static int __init d40_phy_res_init(struct d40_base *base)
2839 {
2840  int i;
2841  int num_phy_chans_avail = 0;
2842  u32 val[2];
2843  int odd_even_bit = -2;
2844  int gcc = D40_DREG_GCC_ENA;
2845 
2846  val[0] = readl(base->virtbase + D40_DREG_PRSME);
2847  val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2848 
2849  for (i = 0; i < base->num_phy_chans; i++) {
2850  base->phy_res[i].num = i;
2851  odd_even_bit += 2 * ((i % 2) == 0);
2852  if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2853  /* Mark security only channels as occupied */
2854  base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2855  base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2856  base->phy_res[i].reserved = true;
2861 
2862 
2863  } else {
2864  base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2865  base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2866  base->phy_res[i].reserved = false;
2867  num_phy_chans_avail++;
2868  }
2869  spin_lock_init(&base->phy_res[i].lock);
2870  }
2871 
2872  /* Mark disabled channels as occupied */
2873  for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2874  int chan = base->plat_data->disabled_channels[i];
2875 
2876  base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2877  base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2878  base->phy_res[chan].reserved = true;
2883  num_phy_chans_avail--;
2884  }
2885 
2886  dev_info(base->dev, "%d of %d physical DMA channels available\n",
2887  num_phy_chans_avail, base->num_phy_chans);
2888 
2889  /* Verify settings extended vs standard */
2890  val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2891 
2892  for (i = 0; i < base->num_phy_chans; i++) {
2893 
2894  if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2895  (val[0] & 0x3) != 1)
2896  dev_info(base->dev,
2897  "[%s] INFO: channel %d is misconfigured (%d)\n",
2898  __func__, i, val[0] & 0x3);
2899 
2900  val[0] = val[0] >> 2;
2901  }
2902 
2903  /*
2904  * To keep things simple, Enable all clocks initially.
2905  * The clocks will get managed later post channel allocation.
2906  * The clocks for the event lines on which reserved channels exists
2907  * are not managed here.
2908  */
2910  base->gcc_pwr_off_mask = gcc;
2911 
2912  return num_phy_chans_avail;
2913 }
2914 
2915 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2916 {
2917  struct stedma40_platform_data *plat_data;
2918  struct clk *clk = NULL;
2919  void __iomem *virtbase = NULL;
2920  struct resource *res = NULL;
2921  struct d40_base *base = NULL;
2922  int num_log_chans = 0;
2923  int num_phy_chans;
2924  int clk_ret = -EINVAL;
2925  int i;
2926  u32 pid;
2927  u32 cid;
2928  u8 rev;
2929 
2930  clk = clk_get(&pdev->dev, NULL);
2931  if (IS_ERR(clk)) {
2932  d40_err(&pdev->dev, "No matching clock found\n");
2933  goto failure;
2934  }
2935 
2936  clk_ret = clk_prepare_enable(clk);
2937  if (clk_ret) {
2938  d40_err(&pdev->dev, "Failed to prepare/enable clock\n");
2939  goto failure;
2940  }
2941 
2942  /* Get IO for DMAC base address */
2943  res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2944  if (!res)
2945  goto failure;
2946 
2947  if (request_mem_region(res->start, resource_size(res),
2948  D40_NAME " I/O base") == NULL)
2949  goto failure;
2950 
2951  virtbase = ioremap(res->start, resource_size(res));
2952  if (!virtbase)
2953  goto failure;
2954 
2955  /* This is just a regular AMBA PrimeCell ID actually */
2956  for (pid = 0, i = 0; i < 4; i++)
2957  pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i)
2958  & 255) << (i * 8);
2959  for (cid = 0, i = 0; i < 4; i++)
2960  cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i)
2961  & 255) << (i * 8);
2962 
2963  if (cid != AMBA_CID) {
2964  d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n");
2965  goto failure;
2966  }
2967  if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
2968  d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
2969  AMBA_MANF_BITS(pid),
2970  AMBA_VENDOR_ST);
2971  goto failure;
2972  }
2973  /*
2974  * HW revision:
2975  * DB8500ed has revision 0
2976  * ? has revision 1
2977  * DB8500v1 has revision 2
2978  * DB8500v2 has revision 3
2979  */
2980  rev = AMBA_REV_BITS(pid);
2981 
2982  /* The number of physical channels on this HW */
2983  num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2984 
2985  dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2986  rev, res->start);
2987 
2988  if (rev < 2) {
2989  d40_err(&pdev->dev, "hardware revision: %d is not supported",
2990  rev);
2991  goto failure;
2992  }
2993 
2994  plat_data = pdev->dev.platform_data;
2995 
2996  /* Count the number of logical channels in use */
2997  for (i = 0; i < plat_data->dev_len; i++)
2998  if (plat_data->dev_rx[i] != 0)
2999  num_log_chans++;
3000 
3001  for (i = 0; i < plat_data->dev_len; i++)
3002  if (plat_data->dev_tx[i] != 0)
3003  num_log_chans++;
3004 
3005  base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
3006  (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
3007  sizeof(struct d40_chan), GFP_KERNEL);
3008 
3009  if (base == NULL) {
3010  d40_err(&pdev->dev, "Out of memory\n");
3011  goto failure;
3012  }
3013 
3014  base->rev = rev;
3015  base->clk = clk;
3016  base->num_phy_chans = num_phy_chans;
3017  base->num_log_chans = num_log_chans;
3018  base->phy_start = res->start;
3019  base->phy_size = resource_size(res);
3020  base->virtbase = virtbase;
3021  base->plat_data = plat_data;
3022  base->dev = &pdev->dev;
3023  base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
3024  base->log_chans = &base->phy_chans[num_phy_chans];
3025 
3026  base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
3027  GFP_KERNEL);
3028  if (!base->phy_res)
3029  goto failure;
3030 
3031  base->lookup_phy_chans = kzalloc(num_phy_chans *
3032  sizeof(struct d40_chan *),
3033  GFP_KERNEL);
3034  if (!base->lookup_phy_chans)
3035  goto failure;
3036 
3037  if (num_log_chans + plat_data->memcpy_len) {
3038  /*
3039  * The max number of logical channels are event lines for all
3040  * src devices and dst devices
3041  */
3042  base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
3043  sizeof(struct d40_chan *),
3044  GFP_KERNEL);
3045  if (!base->lookup_log_chans)
3046  goto failure;
3047  }
3048 
3049  base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
3050  sizeof(d40_backup_regs_chan),
3051  GFP_KERNEL);
3052  if (!base->reg_val_backup_chan)
3053  goto failure;
3054 
3055  base->lcla_pool.alloc_map =
3056  kzalloc(num_phy_chans * sizeof(struct d40_desc *)
3058  if (!base->lcla_pool.alloc_map)
3059  goto failure;
3060 
3061  base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
3062  0, SLAB_HWCACHE_ALIGN,
3063  NULL);
3064  if (base->desc_slab == NULL)
3065  goto failure;
3066 
3067  return base;
3068 
3069 failure:
3070  if (!clk_ret)
3071  clk_disable_unprepare(clk);
3072  if (!IS_ERR(clk))
3073  clk_put(clk);
3074  if (virtbase)
3075  iounmap(virtbase);
3076  if (res)
3078  resource_size(res));
3079  if (virtbase)
3080  iounmap(virtbase);
3081 
3082  if (base) {
3083  kfree(base->lcla_pool.alloc_map);
3084  kfree(base->reg_val_backup_chan);
3085  kfree(base->lookup_log_chans);
3086  kfree(base->lookup_phy_chans);
3087  kfree(base->phy_res);
3088  kfree(base);
3089  }
3090 
3091  return NULL;
3092 }
3093 
3094 static void __init d40_hw_init(struct d40_base *base)
3095 {
3096 
3097  static struct d40_reg_val dma_init_reg[] = {
3098  /* Clock every part of the DMA block from start */
3100 
3101  /* Interrupts on all logical channels */
3102  { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
3103  { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
3104  { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
3105  { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
3106  { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
3107  { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
3108  { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
3109  { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
3110  { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
3111  { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
3112  { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
3113  { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
3114  };
3115  int i;
3116  u32 prmseo[2] = {0, 0};
3117  u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
3118  u32 pcmis = 0;
3119  u32 pcicr = 0;
3120 
3121  for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
3122  writel(dma_init_reg[i].val,
3123  base->virtbase + dma_init_reg[i].reg);
3124 
3125  /* Configure all our dma channels to default settings */
3126  for (i = 0; i < base->num_phy_chans; i++) {
3127 
3128  activeo[i % 2] = activeo[i % 2] << 2;
3129 
3130  if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
3131  == D40_ALLOC_PHY) {
3132  activeo[i % 2] |= 3;
3133  continue;
3134  }
3135 
3136  /* Enable interrupt # */
3137  pcmis = (pcmis << 1) | 1;
3138 
3139  /* Clear interrupt # */
3140  pcicr = (pcicr << 1) | 1;
3141 
3142  /* Set channel to physical mode */
3143  prmseo[i % 2] = prmseo[i % 2] << 2;
3144  prmseo[i % 2] |= 1;
3145 
3146  }
3147 
3148  writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
3149  writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
3150  writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
3151  writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
3152 
3153  /* Write which interrupt to enable */
3154  writel(pcmis, base->virtbase + D40_DREG_PCMIS);
3155 
3156  /* Write which interrupt to clear */
3157  writel(pcicr, base->virtbase + D40_DREG_PCICR);
3158 
3159 }
3160 
3161 static int __init d40_lcla_allocate(struct d40_base *base)
3162 {
3163  struct d40_lcla_pool *pool = &base->lcla_pool;
3164  unsigned long *page_list;
3165  int i, j;
3166  int ret = 0;
3167 
3168  /*
3169  * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
3170  * To full fill this hardware requirement without wasting 256 kb
3171  * we allocate pages until we get an aligned one.
3172  */
3173  page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
3174  GFP_KERNEL);
3175 
3176  if (!page_list) {
3177  ret = -ENOMEM;
3178  goto failure;
3179  }
3180 
3181  /* Calculating how many pages that are required */
3182  base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
3183 
3184  for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
3185  page_list[i] = __get_free_pages(GFP_KERNEL,
3186  base->lcla_pool.pages);
3187  if (!page_list[i]) {
3188 
3189  d40_err(base->dev, "Failed to allocate %d pages.\n",
3190  base->lcla_pool.pages);
3191 
3192  for (j = 0; j < i; j++)
3193  free_pages(page_list[j], base->lcla_pool.pages);
3194  goto failure;
3195  }
3196 
3197  if ((virt_to_phys((void *)page_list[i]) &
3198  (LCLA_ALIGNMENT - 1)) == 0)
3199  break;
3200  }
3201 
3202  for (j = 0; j < i; j++)
3203  free_pages(page_list[j], base->lcla_pool.pages);
3204 
3205  if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
3206  base->lcla_pool.base = (void *)page_list[i];
3207  } else {
3208  /*
3209  * After many attempts and no succees with finding the correct
3210  * alignment, try with allocating a big buffer.
3211  */
3212  dev_warn(base->dev,
3213  "[%s] Failed to get %d pages @ 18 bit align.\n",
3214  __func__, base->lcla_pool.pages);
3215  base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
3216  base->num_phy_chans +
3218  GFP_KERNEL);
3219  if (!base->lcla_pool.base_unaligned) {
3220  ret = -ENOMEM;
3221  goto failure;
3222  }
3223 
3224  base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
3225  LCLA_ALIGNMENT);
3226  }
3227 
3228  pool->dma_addr = dma_map_single(base->dev, pool->base,
3229  SZ_1K * base->num_phy_chans,
3230  DMA_TO_DEVICE);
3231  if (dma_mapping_error(base->dev, pool->dma_addr)) {
3232  pool->dma_addr = 0;
3233  ret = -ENOMEM;
3234  goto failure;
3235  }
3236 
3237  writel(virt_to_phys(base->lcla_pool.base),
3238  base->virtbase + D40_DREG_LCLA);
3239 failure:
3240  kfree(page_list);
3241  return ret;
3242 }
3243 
3244 static int __init d40_probe(struct platform_device *pdev)
3245 {
3246  int err;
3247  int ret = -ENOENT;
3248  struct d40_base *base;
3249  struct resource *res = NULL;
3250  int num_reserved_chans;
3251  u32 val;
3252 
3253  base = d40_hw_detect_init(pdev);
3254 
3255  if (!base)
3256  goto failure;
3257 
3258  num_reserved_chans = d40_phy_res_init(base);
3259 
3260  platform_set_drvdata(pdev, base);
3261 
3263  spin_lock_init(&base->execmd_lock);
3264 
3265  /* Get IO for logical channel parameter address */
3266  res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
3267  if (!res) {
3268  ret = -ENOENT;
3269  d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
3270  goto failure;
3271  }
3272  base->lcpa_size = resource_size(res);
3273  base->phy_lcpa = res->start;
3274 
3275  if (request_mem_region(res->start, resource_size(res),
3276  D40_NAME " I/O lcpa") == NULL) {
3277  ret = -EBUSY;
3278  d40_err(&pdev->dev,
3279  "Failed to request LCPA region 0x%x-0x%x\n",
3280  res->start, res->end);
3281  goto failure;
3282  }
3283 
3284  /* We make use of ESRAM memory for this. */
3285  val = readl(base->virtbase + D40_DREG_LCPA);
3286  if (res->start != val && val != 0) {
3287  dev_warn(&pdev->dev,
3288  "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
3289  __func__, val, res->start);
3290  } else
3291  writel(res->start, base->virtbase + D40_DREG_LCPA);
3292 
3293  base->lcpa_base = ioremap(res->start, resource_size(res));
3294  if (!base->lcpa_base) {
3295  ret = -ENOMEM;
3296  d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
3297  goto failure;
3298  }
3299  /* If lcla has to be located in ESRAM we don't need to allocate */
3300  if (base->plat_data->use_esram_lcla) {
3302  "lcla_esram");
3303  if (!res) {
3304  ret = -ENOENT;
3305  d40_err(&pdev->dev,
3306  "No \"lcla_esram\" memory resource\n");
3307  goto failure;
3308  }
3309  base->lcla_pool.base = ioremap(res->start,
3310  resource_size(res));
3311  if (!base->lcla_pool.base) {
3312  ret = -ENOMEM;
3313  d40_err(&pdev->dev, "Failed to ioremap LCLA region\n");
3314  goto failure;
3315  }
3316  writel(res->start, base->virtbase + D40_DREG_LCLA);
3317 
3318  } else {
3319  ret = d40_lcla_allocate(base);
3320  if (ret) {
3321  d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
3322  goto failure;
3323  }
3324  }
3325 
3326  spin_lock_init(&base->lcla_pool.lock);
3327 
3328  base->irq = platform_get_irq(pdev, 0);
3329 
3330  ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
3331  if (ret) {
3332  d40_err(&pdev->dev, "No IRQ defined\n");
3333  goto failure;
3334  }
3335 
3336  pm_runtime_irq_safe(base->dev);
3338  pm_runtime_use_autosuspend(base->dev);
3339  pm_runtime_enable(base->dev);
3340  pm_runtime_resume(base->dev);
3341 
3342  if (base->plat_data->use_esram_lcla) {
3343 
3344  base->lcpa_regulator = regulator_get(base->dev, "lcla_esram");
3345  if (IS_ERR(base->lcpa_regulator)) {
3346  d40_err(&pdev->dev, "Failed to get lcpa_regulator\n");
3347  base->lcpa_regulator = NULL;
3348  goto failure;
3349  }
3350 
3351  ret = regulator_enable(base->lcpa_regulator);
3352  if (ret) {
3353  d40_err(&pdev->dev,
3354  "Failed to enable lcpa_regulator\n");
3356  base->lcpa_regulator = NULL;
3357  goto failure;
3358  }
3359  }
3360 
3361  base->initialized = true;
3362  err = d40_dmaengine_init(base, num_reserved_chans);
3363  if (err)
3364  goto failure;
3365 
3366  d40_hw_init(base);
3367 
3368  dev_info(base->dev, "initialized\n");
3369  return 0;
3370 
3371 failure:
3372  if (base) {
3373  if (base->desc_slab)
3375  if (base->virtbase)
3376  iounmap(base->virtbase);
3377 
3378  if (base->lcla_pool.base && base->plat_data->use_esram_lcla) {
3379  iounmap(base->lcla_pool.base);
3380  base->lcla_pool.base = NULL;
3381  }
3382 
3383  if (base->lcla_pool.dma_addr)
3384  dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
3385  SZ_1K * base->num_phy_chans,
3386  DMA_TO_DEVICE);
3387 
3388  if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3389  free_pages((unsigned long)base->lcla_pool.base,
3390  base->lcla_pool.pages);
3391 
3392  kfree(base->lcla_pool.base_unaligned);
3393 
3394  if (base->phy_lcpa)
3396  base->lcpa_size);
3397  if (base->phy_start)
3399  base->phy_size);
3400  if (base->clk) {
3401  clk_disable(base->clk);
3402  clk_put(base->clk);
3403  }
3404 
3405  if (base->lcpa_regulator) {
3408  }
3409 
3410  kfree(base->lcla_pool.alloc_map);
3411  kfree(base->lookup_log_chans);
3412  kfree(base->lookup_phy_chans);
3413  kfree(base->phy_res);
3414  kfree(base);
3415  }
3416 
3417  d40_err(&pdev->dev, "probe failed\n");
3418  return ret;
3419 }
3420 
3421 static struct platform_driver d40_driver = {
3422  .driver = {
3423  .owner = THIS_MODULE,
3424  .name = D40_NAME,
3425  .pm = DMA40_PM_OPS,
3426  },
3427 };
3428 
3429 static int __init stedma40_init(void)
3430 {
3431  return platform_driver_probe(&d40_driver, d40_probe);
3432 }
3433 subsys_initcall(stedma40_init);