Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
smtc.c
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software
14  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15  *
16  * Copyright (C) 2004 Mips Technologies, Inc
17  * Copyright (C) 2008 Kevin D. Kissell
18  */
19 
20 #include <linux/clockchips.h>
21 #include <linux/kernel.h>
22 #include <linux/sched.h>
23 #include <linux/smp.h>
24 #include <linux/cpumask.h>
25 #include <linux/interrupt.h>
26 #include <linux/kernel_stat.h>
27 #include <linux/module.h>
28 #include <linux/ftrace.h>
29 #include <linux/slab.h>
30 
31 #include <asm/cpu.h>
32 #include <asm/processor.h>
33 #include <linux/atomic.h>
34 #include <asm/hardirq.h>
35 #include <asm/hazards.h>
36 #include <asm/irq.h>
37 #include <asm/mmu_context.h>
38 #include <asm/mipsregs.h>
39 #include <asm/cacheflush.h>
40 #include <asm/time.h>
41 #include <asm/addrspace.h>
42 #include <asm/smtc.h>
43 #include <asm/smtc_proc.h>
44 
45 /*
46  * SMTC Kernel needs to manipulate low-level CPU interrupt mask
47  * in do_IRQ. These are passed in setup_irq_smtc() and stored
48  * in this table.
49  */
50 unsigned long irq_hwmask[NR_IRQS];
51 
52 #define LOCK_MT_PRA() \
53  local_irq_save(flags); \
54  mtflags = dmt()
55 
56 #define UNLOCK_MT_PRA() \
57  emt(mtflags); \
58  local_irq_restore(flags)
59 
60 #define LOCK_CORE_PRA() \
61  local_irq_save(flags); \
62  mtflags = dvpe()
63 
64 #define UNLOCK_CORE_PRA() \
65  evpe(mtflags); \
66  local_irq_restore(flags)
67 
68 /*
69  * Data structures purely associated with SMTC parallelism
70  */
71 
72 
73 /*
74  * Table for tracking ASIDs whose lifetime is prolonged.
75  */
76 
78 
79 /*
80  * Number of InterProcessor Interrupt (IPI) message buffers to allocate
81  */
82 
83 #define IPIBUF_PER_CPU 4
84 
86 static struct smtc_ipi_q freeIPIq;
87 
88 
89 /*
90  * Number of FPU contexts for each VPE
91  */
92 
93 static int smtc_nconf1[MAX_SMTC_VPES];
94 
95 
96 /* Forward declarations */
97 
98 void ipi_decode(struct smtc_ipi *);
99 static void post_direct_ipi(int cpu, struct smtc_ipi *pipi);
100 static void setup_cross_vpe_interrupts(unsigned int nvpe);
101 void init_smtc_stats(void);
102 
103 /* Global SMTC Status */
104 
105 unsigned int smtc_status;
106 
107 /* Boot command line configuration overrides */
108 
109 static int vpe0limit;
110 static int ipibuffers;
111 static int nostlb;
112 static int asidmask;
113 unsigned long smtc_asid_mask = 0xff;
114 
115 static int __init vpe0tcs(char *str)
116 {
117  get_option(&str, &vpe0limit);
118 
119  return 1;
120 }
121 
122 static int __init ipibufs(char *str)
123 {
124  get_option(&str, &ipibuffers);
125  return 1;
126 }
127 
128 static int __init stlb_disable(char *s)
129 {
130  nostlb = 1;
131  return 1;
132 }
133 
134 static int __init asidmask_set(char *str)
135 {
136  get_option(&str, &asidmask);
137  switch (asidmask) {
138  case 0x1:
139  case 0x3:
140  case 0x7:
141  case 0xf:
142  case 0x1f:
143  case 0x3f:
144  case 0x7f:
145  case 0xff:
146  smtc_asid_mask = (unsigned long)asidmask;
147  break;
148  default:
149  printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask);
150  }
151  return 1;
152 }
153 
154 __setup("vpe0tcs=", vpe0tcs);
155 __setup("ipibufs=", ipibufs);
156 __setup("nostlb", stlb_disable);
157 __setup("asidmask=", asidmask_set);
158 
159 #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
160 
161 static int hang_trig;
162 
163 static int __init hangtrig_enable(char *s)
164 {
165  hang_trig = 1;
166  return 1;
167 }
168 
169 
170 __setup("hangtrig", hangtrig_enable);
171 
172 #define DEFAULT_BLOCKED_IPI_LIMIT 32
173 
174 static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT;
175 
176 static int __init tintq(char *str)
177 {
178  get_option(&str, &timerq_limit);
179  return 1;
180 }
181 
182 __setup("tintq=", tintq);
183 
184 static int imstuckcount[MAX_SMTC_VPES][8];
185 /* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */
186 static int vpemask[MAX_SMTC_VPES][8] = {
187  {0, 0, 1, 0, 0, 0, 0, 1},
188  {0, 0, 0, 0, 0, 0, 0, 1}
189 };
190 int tcnoprog[NR_CPUS];
191 static atomic_t idle_hook_initialized = ATOMIC_INIT(0);
192 static int clock_hang_reported[NR_CPUS];
193 
194 #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
195 
196 /*
197  * Configure shared TLB - VPC configuration bit must be set by caller
198  */
199 
200 static void smtc_configure_tlb(void)
201 {
202  int i, tlbsiz, vpes;
203  unsigned long mvpconf0;
204  unsigned long config1val;
205 
206  /* Set up ASID preservation table */
207  for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) {
208  for(i = 0; i < MAX_SMTC_ASIDS; i++) {
209  smtc_live_asid[vpes][i] = 0;
210  }
211  }
212  mvpconf0 = read_c0_mvpconf0();
213 
214  if ((vpes = ((mvpconf0 & MVPCONF0_PVPE)
215  >> MVPCONF0_PVPE_SHIFT) + 1) > 1) {
216  /* If we have multiple VPEs, try to share the TLB */
217  if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) {
218  /*
219  * If TLB sizing is programmable, shared TLB
220  * size is the total available complement.
221  * Otherwise, we have to take the sum of all
222  * static VPE TLB entries.
223  */
224  if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE)
225  >> MVPCONF0_PTLBE_SHIFT)) == 0) {
226  /*
227  * If there's more than one VPE, there had better
228  * be more than one TC, because we need one to bind
229  * to each VPE in turn to be able to read
230  * its configuration state!
231  */
232  settc(1);
233  /* Stop the TC from doing anything foolish */
235  mips_ihb();
236  /* No need to un-Halt - that happens later anyway */
237  for (i=0; i < vpes; i++) {
239  /*
240  * To be 100% sure we're really getting the right
241  * information, we exit the configuration state
242  * and do an IHB after each rebinding.
243  */
246  mips_ihb();
247  /*
248  * Only count if the MMU Type indicated is TLB
249  */
250  if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) {
251  config1val = read_vpe_c0_config1();
252  tlbsiz += ((config1val >> 25) & 0x3f) + 1;
253  }
254 
255  /* Put core back in configuration state */
258  mips_ihb();
259  }
260  }
262  ehb();
263 
264  /*
265  * Setup kernel data structures to use software total,
266  * rather than read the per-VPE Config1 value. The values
267  * for "CPU 0" gets copied to all the other CPUs as part
268  * of their initialization in smtc_cpu_setup().
269  */
270 
271  /* MIPS32 limits TLB indices to 64 */
272  if (tlbsiz > 64)
273  tlbsiz = 64;
274  cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz;
277 
278  printk("TLB of %d entry pairs shared by %d VPEs\n",
279  tlbsiz, vpes);
280  } else {
281  printk("WARNING: TLB Not Sharable on SMTC Boot!\n");
282  }
283  }
284 }
285 
286 
287 /*
288  * Incrementally build the CPU map out of constituent MIPS MT cores,
289  * using the specified available VPEs and TCs. Plaform code needs
290  * to ensure that each MIPS MT core invokes this routine on reset,
291  * one at a time(!).
292  *
293  * This version of the build_cpu_map and prepare_cpus routines assumes
294  * that *all* TCs of a MIPS MT core will be used for Linux, and that
295  * they will be spread across *all* available VPEs (to minimise the
296  * loss of efficiency due to exception service serialization).
297  * An improved version would pick up configuration information and
298  * possibly leave some TCs/VPEs as "slave" processors.
299  *
300  * Use c0_MVPConf0 to find out how many TCs are available, setting up
301  * cpu_possible_mask and the logical/physical mappings.
302  */
303 
304 int __init smtc_build_cpu_map(int start_cpu_slot)
305 {
306  int i, ntcs;
307 
308  /*
309  * The CPU map isn't actually used for anything at this point,
310  * so it's not clear what else we should do apart from set
311  * everything up so that "logical" = "physical".
312  */
313  ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
314  for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) {
315  set_cpu_possible(i, true);
316  __cpu_number_map[i] = i;
317  __cpu_logical_map[i] = i;
318  }
319 #ifdef CONFIG_MIPS_MT_FPAFF
320  /* Initialize map of CPUs with FPUs */
322 #endif
323 
324  /* One of those TC's is the one booting, and not a secondary... */
325  printk("%i available secondary CPU TC(s)\n", i - 1);
326 
327  return i;
328 }
329 
330 /*
331  * Common setup before any secondaries are started
332  * Make sure all CPUs are in a sensible state before we boot any of the
333  * secondaries.
334  *
335  * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly
336  * as possible across the available VPEs.
337  */
338 
339 static void smtc_tc_setup(int vpe, int tc, int cpu)
340 {
341  static int cp1contexts[MAX_SMTC_VPES];
342 
343  /*
344  * Make a local copy of the available FPU contexts in order
345  * to keep track of TCs that can have one.
346  */
347  if (tc == 1)
348  {
349  /*
350  * FIXME: Multi-core SMTC hasn't been tested and the
351  * maximum number of VPEs may change.
352  */
353  cp1contexts[0] = smtc_nconf1[0] - 1;
354  cp1contexts[1] = smtc_nconf1[1];
355  }
356 
357  settc(tc);
359  mips_ihb();
362  | TCSTATUS_A);
363  /*
364  * TCContext gets an offset from the base of the IPIQ array
365  * to be used in low-level code to detect the presence of
366  * an active IPI queue.
367  */
368  write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16);
369 
370  /* Bind TC to VPE. */
371  write_tc_c0_tcbind(vpe);
372 
373  /* In general, all TCs should have the same cpu_data indications. */
374  memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips));
375 
376  /* Check to see if there is a FPU context available for this TC. */
377  if (!cp1contexts[vpe])
378  cpu_data[cpu].options &= ~MIPS_CPU_FPU;
379  else
380  cp1contexts[vpe]--;
381 
382  /* Store the TC and VPE into the cpu_data structure. */
383  cpu_data[cpu].vpe_id = vpe;
384  cpu_data[cpu].tc_id = tc;
385 
386  /* FIXME: Multi-core SMTC hasn't been tested, but be prepared. */
387  cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff;
388 }
389 
390 /*
391  * Tweak to get Count registers synced as closely as possible. The
392  * value seems good for 34K-class cores.
393  */
394 
395 #define CP0_SKEW 8
396 
398 {
399  int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu;
400  unsigned long flags;
401  unsigned long val;
402  int nipi;
403  struct smtc_ipi *pipi;
404 
405  /* disable interrupts so we can disable MT */
406  local_irq_save(flags);
407  /* disable MT so we can configure */
408  dvpe();
409  dmt();
410 
411  spin_lock_init(&freeIPIq.lock);
412 
413  /*
414  * We probably don't have as many VPEs as we do SMP "CPUs",
415  * but it's possible - and in any case we'll never use more!
416  */
417  for (i=0; i<NR_CPUS; i++) {
418  IPIQ[i].head = IPIQ[i].tail = NULL;
419  spin_lock_init(&IPIQ[i].lock);
420  IPIQ[i].depth = 0;
421  IPIQ[i].resched_flag = 0; /* No reschedules queued initially */
422  }
423 
424  /* cpu_data index starts at zero */
425  cpu = 0;
426  cpu_data[cpu].vpe_id = 0;
427  cpu_data[cpu].tc_id = 0;
428  cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff;
429  cpu++;
430 
431  /* Report on boot-time options */
433  if (vpelimit > 0)
434  printk("Limit of %d VPEs set\n", vpelimit);
435  if (tclimit > 0)
436  printk("Limit of %d TCs set\n", tclimit);
437  if (nostlb) {
438  printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n");
439  }
440  if (asidmask)
441  printk("ASID mask value override to 0x%x\n", asidmask);
442 
443  /* Temporary */
444 #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
445  if (hang_trig)
446  printk("Logic Analyser Trigger on suspected TC hang\n");
447 #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
448 
449  /* Put MVPE's into 'configuration state' */
451 
452  val = read_c0_mvpconf0();
453  nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1;
454  if (vpelimit > 0 && nvpe > vpelimit)
455  nvpe = vpelimit;
456  ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
457  if (ntc > NR_CPUS)
458  ntc = NR_CPUS;
459  if (tclimit > 0 && ntc > tclimit)
460  ntc = tclimit;
461  slop = ntc % nvpe;
462  for (i = 0; i < nvpe; i++) {
463  tcpervpe[i] = ntc / nvpe;
464  if (slop) {
465  if((slop - i) > 0) tcpervpe[i]++;
466  }
467  }
468  /* Handle command line override for VPE0 */
469  if (vpe0limit > ntc) vpe0limit = ntc;
470  if (vpe0limit > 0) {
471  int slopslop;
472  if (vpe0limit < tcpervpe[0]) {
473  /* Reducing TC count - distribute to others */
474  slop = tcpervpe[0] - vpe0limit;
475  slopslop = slop % (nvpe - 1);
476  tcpervpe[0] = vpe0limit;
477  for (i = 1; i < nvpe; i++) {
478  tcpervpe[i] += slop / (nvpe - 1);
479  if(slopslop && ((slopslop - (i - 1) > 0)))
480  tcpervpe[i]++;
481  }
482  } else if (vpe0limit > tcpervpe[0]) {
483  /* Increasing TC count - steal from others */
484  slop = vpe0limit - tcpervpe[0];
485  slopslop = slop % (nvpe - 1);
486  tcpervpe[0] = vpe0limit;
487  for (i = 1; i < nvpe; i++) {
488  tcpervpe[i] -= slop / (nvpe - 1);
489  if(slopslop && ((slopslop - (i - 1) > 0)))
490  tcpervpe[i]--;
491  }
492  }
493  }
494 
495  /* Set up shared TLB */
496  smtc_configure_tlb();
497 
498  for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) {
499  /* Get number of CP1 contexts for each VPE. */
500  if (tc == 0)
501  {
502  /*
503  * Do not call settc() for TC0 or the FPU context
504  * value will be incorrect. Besides, we know that
505  * we are TC0 anyway.
506  */
507  smtc_nconf1[0] = ((read_vpe_c0_vpeconf1() &
509  if (nvpe == 2)
510  {
511  settc(1);
512  smtc_nconf1[1] = ((read_vpe_c0_vpeconf1() &
514  settc(0);
515  }
516  }
517  if (tcpervpe[vpe] == 0)
518  continue;
519  if (vpe != 0)
520  printk(", ");
521  printk("VPE %d: TC", vpe);
522  for (i = 0; i < tcpervpe[vpe]; i++) {
523  /*
524  * TC 0 is bound to VPE 0 at reset,
525  * and is presumably executing this
526  * code. Leave it alone!
527  */
528  if (tc != 0) {
529  smtc_tc_setup(vpe, tc, cpu);
530  if (vpe != 0) {
531  /*
532  * Set MVP bit (possibly again). Do it
533  * here to catch CPUs that have no TCs
534  * bound to the VPE at reset. In that
535  * case, a TC must be bound to the VPE
536  * before we can set VPEControl[MVP]
537  */
540  VPECONF0_MVP);
541  }
542  cpu++;
543  }
544  printk(" %d", tc);
545  tc++;
546  }
547  if (vpe != 0) {
548  /*
549  * Allow this VPE to control others.
550  */
552  VPECONF0_MVP);
553 
554  /*
555  * Clear any stale software interrupts from VPE's Cause
556  */
558 
559  /*
560  * Clear ERL/EXL of VPEs other than 0
561  * and set restricted interrupt enable/mask.
562  */
564  & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM))
566  | ST0_IE));
567  /*
568  * set config to be the same as vpe0,
569  * particularly kseg0 coherency alg
570  */
572  /* Clear any pending timer interrupt */
574  /* Propagate Config7 */
577  ehb();
578  }
579  /* enable multi-threading within VPE */
581  /* enable the VPE */
583  }
584 
585  /*
586  * Pull any physically present but unused TCs out of circulation.
587  */
588  while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) {
589  set_cpu_possible(tc, false);
590  set_cpu_present(tc, false);
591  tc++;
592  }
593 
594  /* release config state */
596 
597  printk("\n");
598 
599  /* Set up coprocessor affinity CPU mask(s) */
600 
601 #ifdef CONFIG_MIPS_MT_FPAFF
602  for (tc = 0; tc < ntc; tc++) {
603  if (cpu_data[tc].options & MIPS_CPU_FPU)
604  cpu_set(tc, mt_fpu_cpumask);
605  }
606 #endif
607 
608  /* set up ipi interrupts... */
609 
610  /* If we have multiple VPEs running, set up the cross-VPE interrupt */
611 
612  setup_cross_vpe_interrupts(nvpe);
613 
614  /* Set up queue of free IPI "messages". */
615  nipi = NR_CPUS * IPIBUF_PER_CPU;
616  if (ipibuffers > 0)
617  nipi = ipibuffers;
618 
619  pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL);
620  if (pipi == NULL)
621  panic("kmalloc of IPI message buffers failed");
622  else
623  printk("IPI buffer pool of %d buffers\n", nipi);
624  for (i = 0; i < nipi; i++) {
625  smtc_ipi_nq(&freeIPIq, pipi);
626  pipi++;
627  }
628 
629  /* Arm multithreading and enable other VPEs - but all TCs are Halted */
630  emt(EMT_ENABLE);
631  evpe(EVPE_ENABLE);
632  local_irq_restore(flags);
633  /* Initialize SMTC /proc statistics/diagnostics */
634  init_smtc_stats();
635 }
636 
637 
638 /*
639  * Setup the PC, SP, and GP of a secondary processor and start it
640  * running!
641  * smp_bootstrap is the place to resume from
642  * __KSTK_TOS(idle) is apparently the stack pointer
643  * (unsigned long)idle->thread_info the gp
644  *
645  */
647 {
648  extern u32 kernelsp[NR_CPUS];
649  unsigned long flags;
650  int mtflags;
651 
652  LOCK_MT_PRA();
654  dvpe();
655  }
656  settc(cpu_data[cpu].tc_id);
657 
658  /* pc */
659  write_tc_c0_tcrestart((unsigned long)&smp_bootstrap);
660 
661  /* stack pointer */
662  kernelsp[cpu] = __KSTK_TOS(idle);
664 
665  /* global pointer */
666  write_tc_gpr_gp((unsigned long)task_thread_info(idle));
667 
671  evpe(EVPE_ENABLE);
672  }
673  UNLOCK_MT_PRA();
674 }
675 
677 {
678 }
679 
680 void smtc_smp_finish(void)
681 {
682  int cpu = smp_processor_id();
683 
684  /*
685  * Lowest-numbered CPU per VPE starts a clock tick.
686  * Like per_cpu_trap_init() hack, this assumes that
687  * SMTC init code assigns TCs consdecutively and
688  * in ascending order across available VPEs.
689  */
690  if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id))
692 
694 
695  printk("TC %d going on-line as CPU %d\n",
697 }
698 
699 void smtc_cpus_done(void)
700 {
701 }
702 
703 /*
704  * Support for SMTC-optimized driver IRQ registration
705  */
706 
707 /*
708  * SMTC Kernel needs to manipulate low-level CPU interrupt mask
709  * in do_IRQ. These are passed in setup_irq_smtc() and stored
710  * in this table.
711  */
712 
713 int setup_irq_smtc(unsigned int irq, struct irqaction * new,
714  unsigned long hwmask)
715 {
716 #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
717  unsigned int vpe = current_cpu_data.vpe_id;
718 
719  vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1;
720 #endif
721  irq_hwmask[irq] = hwmask;
722 
723  return setup_irq(irq, new);
724 }
725 
726 #ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
727 /*
728  * Support for IRQ affinity to TCs
729  */
730 
731 void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity)
732 {
733  /*
734  * If a "fast path" cache of quickly decodable affinity state
735  * is maintained, this is where it gets done, on a call up
736  * from the platform affinity code.
737  */
738 }
739 
740 void smtc_forward_irq(struct irq_data *d)
741 {
742  unsigned int irq = d->irq;
743  int target;
744 
745  /*
746  * OK wise guy, now figure out how to get the IRQ
747  * to be serviced on an authorized "CPU".
748  *
749  * Ideally, to handle the situation where an IRQ has multiple
750  * eligible CPUS, we would maintain state per IRQ that would
751  * allow a fair distribution of service requests. Since the
752  * expected use model is any-or-only-one, for simplicity
753  * and efficiency, we just pick the easiest one to find.
754  */
755 
756  target = cpumask_first(d->affinity);
757 
758  /*
759  * We depend on the platform code to have correctly processed
760  * IRQ affinity change requests to ensure that the IRQ affinity
761  * mask has been purged of bits corresponding to nonexistent and
762  * offline "CPUs", and to TCs bound to VPEs other than the VPE
763  * connected to the physical interrupt input for the interrupt
764  * in question. Otherwise we have a nasty problem with interrupt
765  * mask management. This is best handled in non-performance-critical
766  * platform IRQ affinity setting code, to minimize interrupt-time
767  * checks.
768  */
769 
770  /* If no one is eligible, service locally */
771  if (target >= NR_CPUS)
772  do_IRQ_no_affinity(irq);
773  else
774  smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq);
775 }
776 
777 #endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */
778 
779 /*
780  * IPI model for SMTC is tricky, because interrupts aren't TC-specific.
781  * Within a VPE one TC can interrupt another by different approaches.
782  * The easiest to get right would probably be to make all TCs except
783  * the target IXMT and set a software interrupt, but an IXMT-based
784  * scheme requires that a handler must run before a new IPI could
785  * be sent, which would break the "broadcast" loops in MIPS MT.
786  * A more gonzo approach within a VPE is to halt the TC, extract
787  * its Restart, Status, and a couple of GPRs, and program the Restart
788  * address to emulate an interrupt.
789  *
790  * Within a VPE, one can be confident that the target TC isn't in
791  * a critical EXL state when halted, since the write to the Halt
792  * register could not have issued on the writing thread if the
793  * halting thread had EXL set. So k0 and k1 of the target TC
794  * can be used by the injection code. Across VPEs, one can't
795  * be certain that the target TC isn't in a critical exception
796  * state. So we try a two-step process of sending a software
797  * interrupt to the target VPE, which either handles the event
798  * itself (if it was the target) or injects the event within
799  * the VPE.
800  */
801 
802 static void smtc_ipi_qdump(void)
803 {
804  int i;
805  struct smtc_ipi *temp;
806 
807  for (i = 0; i < NR_CPUS ;i++) {
808  pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n",
809  i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail,
810  IPIQ[i].depth);
811  temp = IPIQ[i].head;
812 
813  while (temp != IPIQ[i].tail) {
814  pr_debug("%d %d %d: ", temp->type, temp->dest,
815  (int)temp->arg);
816 #ifdef SMTC_IPI_DEBUG
817  pr_debug("%u %lu\n", temp->sender, temp->stamp);
818 #else
819  pr_debug("\n");
820 #endif
821  temp = temp->flink;
822  }
823  }
824 }
825 
826 /*
827  * The standard atomic.h primitives don't quite do what we want
828  * here: We need an atomic add-and-return-previous-value (which
829  * could be done with atomic_add_return and a decrement) and an
830  * atomic set/zero-and-return-previous-value (which can't really
831  * be done with the atomic.h primitives). And since this is
832  * MIPS MT, we can assume that we have LL/SC.
833  */
834 static inline int atomic_postincrement(atomic_t *v)
835 {
836  unsigned long result;
837 
838  unsigned long temp;
839 
840  __asm__ __volatile__(
841  "1: ll %0, %2 \n"
842  " addu %1, %0, 1 \n"
843  " sc %1, %2 \n"
844  " beqz %1, 1b \n"
846  : "=&r" (result), "=&r" (temp), "=m" (v->counter)
847  : "m" (v->counter)
848  : "memory");
849 
850  return result;
851 }
852 
853 void smtc_send_ipi(int cpu, int type, unsigned int action)
854 {
855  int tcstatus;
856  struct smtc_ipi *pipi;
857  unsigned long flags;
858  int mtflags;
859  unsigned long tcrestart;
860  extern void r4k_wait_irqoff(void), __pastwait(void);
861  int set_resched_flag = (type == LINUX_SMP_IPI &&
862  action == SMP_RESCHEDULE_YOURSELF);
863 
864  if (cpu == smp_processor_id()) {
865  printk("Cannot Send IPI to self!\n");
866  return;
867  }
868  if (set_resched_flag && IPIQ[cpu].resched_flag != 0)
869  return; /* There is a reschedule queued already */
870 
871  /* Set up a descriptor, to be delivered either promptly or queued */
872  pipi = smtc_ipi_dq(&freeIPIq);
873  if (pipi == NULL) {
874  bust_spinlocks(1);
875  mips_mt_regdump(dvpe());
876  panic("IPI Msg. Buffers Depleted");
877  }
878  pipi->type = type;
879  pipi->arg = (void *)action;
880  pipi->dest = cpu;
881  if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
882  /* If not on same VPE, enqueue and send cross-VPE interrupt */
883  IPIQ[cpu].resched_flag |= set_resched_flag;
884  smtc_ipi_nq(&IPIQ[cpu], pipi);
885  LOCK_CORE_PRA();
886  settc(cpu_data[cpu].tc_id);
888  UNLOCK_CORE_PRA();
889  } else {
890  /*
891  * Not sufficient to do a LOCK_MT_PRA (dmt) here,
892  * since ASID shootdown on the other VPE may
893  * collide with this operation.
894  */
895  LOCK_CORE_PRA();
896  settc(cpu_data[cpu].tc_id);
897  /* Halt the targeted TC */
899  mips_ihb();
900 
901  /*
902  * Inspect TCStatus - if IXMT is set, we have to queue
903  * a message. Otherwise, we set up the "interrupt"
904  * of the other TC
905  */
906  tcstatus = read_tc_c0_tcstatus();
907 
908  if ((tcstatus & TCSTATUS_IXMT) != 0) {
909  /*
910  * If we're in the the irq-off version of the wait
911  * loop, we need to force exit from the wait and
912  * do a direct post of the IPI.
913  */
914  if (cpu_wait == r4k_wait_irqoff) {
915  tcrestart = read_tc_c0_tcrestart();
916  if (tcrestart >= (unsigned long)r4k_wait_irqoff
917  && tcrestart < (unsigned long)__pastwait) {
918  write_tc_c0_tcrestart(__pastwait);
919  tcstatus &= ~TCSTATUS_IXMT;
920  write_tc_c0_tcstatus(tcstatus);
921  goto postdirect;
922  }
923  }
924  /*
925  * Otherwise we queue the message for the target TC
926  * to pick up when he does a local_irq_restore()
927  */
929  UNLOCK_CORE_PRA();
930  IPIQ[cpu].resched_flag |= set_resched_flag;
931  smtc_ipi_nq(&IPIQ[cpu], pipi);
932  } else {
933 postdirect:
934  post_direct_ipi(cpu, pipi);
936  UNLOCK_CORE_PRA();
937  }
938  }
939 }
940 
941 /*
942  * Send IPI message to Halted TC, TargTC/TargVPE already having been set
943  */
944 static void post_direct_ipi(int cpu, struct smtc_ipi *pipi)
945 {
946  struct pt_regs *kstack;
947  unsigned long tcstatus;
948  unsigned long tcrestart;
949  extern u32 kernelsp[NR_CPUS];
950  extern void __smtc_ipi_vector(void);
951 //printk("%s: on %d for %d\n", __func__, smp_processor_id(), cpu);
952 
953  /* Extract Status, EPC from halted TC */
954  tcstatus = read_tc_c0_tcstatus();
955  tcrestart = read_tc_c0_tcrestart();
956  /* If TCRestart indicates a WAIT instruction, advance the PC */
957  if ((tcrestart & 0x80000000)
958  && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) {
959  tcrestart += 4;
960  }
961  /*
962  * Save on TC's future kernel stack
963  *
964  * CU bit of Status is indicator that TC was
965  * already running on a kernel stack...
966  */
967  if (tcstatus & ST0_CU0) {
968  /* Note that this "- 1" is pointer arithmetic */
969  kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1;
970  } else {
971  kstack = ((struct pt_regs *)kernelsp[cpu]) - 1;
972  }
973 
974  kstack->cp0_epc = (long)tcrestart;
975  /* Save TCStatus */
976  kstack->cp0_tcstatus = tcstatus;
977  /* Pass token of operation to be performed kernel stack pad area */
978  kstack->pad0[4] = (unsigned long)pipi;
979  /* Pass address of function to be called likewise */
980  kstack->pad0[5] = (unsigned long)&ipi_decode;
981  /* Set interrupt exempt and kernel mode */
982  tcstatus |= TCSTATUS_IXMT;
983  tcstatus &= ~TCSTATUS_TKSU;
984  write_tc_c0_tcstatus(tcstatus);
985  ehb();
986  /* Set TC Restart address to be SMTC IPI vector */
987  write_tc_c0_tcrestart(__smtc_ipi_vector);
988 }
989 
990 static void ipi_resched_interrupt(void)
991 {
992  scheduler_ipi();
993 }
994 
995 static void ipi_call_interrupt(void)
996 {
997  /* Invoke generic function invocation code in smp.c */
999 }
1000 
1001 DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device);
1002 
1003 static void __irq_entry smtc_clock_tick_interrupt(void)
1004 {
1005  unsigned int cpu = smp_processor_id();
1006  struct clock_event_device *cd;
1007  int irq = MIPS_CPU_IRQ_BASE + 1;
1008 
1009  irq_enter();
1010  kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
1011  cd = &per_cpu(mips_clockevent_device, cpu);
1012  cd->event_handler(cd);
1013  irq_exit();
1014 }
1015 
1016 void ipi_decode(struct smtc_ipi *pipi)
1017 {
1018  void *arg_copy = pipi->arg;
1019  int type_copy = pipi->type;
1020 
1021  smtc_ipi_nq(&freeIPIq, pipi);
1022 
1023  switch (type_copy) {
1024  case SMTC_CLOCK_TICK:
1025  smtc_clock_tick_interrupt();
1026  break;
1027 
1028  case LINUX_SMP_IPI:
1029  switch ((int)arg_copy) {
1031  ipi_resched_interrupt();
1032  break;
1033  case SMP_CALL_FUNCTION:
1034  ipi_call_interrupt();
1035  break;
1036  default:
1037  printk("Impossible SMTC IPI Argument %p\n", arg_copy);
1038  break;
1039  }
1040  break;
1041 #ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
1042  case IRQ_AFFINITY_IPI:
1043  /*
1044  * Accept a "forwarded" interrupt that was initially
1045  * taken by a TC who doesn't have affinity for the IRQ.
1046  */
1047  do_IRQ_no_affinity((int)arg_copy);
1048  break;
1049 #endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */
1050  default:
1051  printk("Impossible SMTC IPI Type 0x%x\n", type_copy);
1052  break;
1053  }
1054 }
1055 
1056 /*
1057  * Similar to smtc_ipi_replay(), but invoked from context restore,
1058  * so it reuses the current exception frame rather than set up a
1059  * new one with self_ipi.
1060  */
1061 
1063 {
1064  int cpu = smp_processor_id();
1065 
1066  /*
1067  * Test is not atomic, but much faster than a dequeue,
1068  * and the vast majority of invocations will have a null queue.
1069  * If irq_disabled when this was called, then any IPIs queued
1070  * after we test last will be taken on the next irq_enable/restore.
1071  * If interrupts were enabled, then any IPIs added after the
1072  * last test will be taken directly.
1073  */
1074 
1075  while (IPIQ[cpu].head != NULL) {
1076  struct smtc_ipi_q *q = &IPIQ[cpu];
1077  struct smtc_ipi *pipi;
1078  unsigned long flags;
1079 
1080  /*
1081  * It may be possible we'll come in with interrupts
1082  * already enabled.
1083  */
1084  local_irq_save(flags);
1085  spin_lock(&q->lock);
1086  pipi = __smtc_ipi_dq(q);
1087  spin_unlock(&q->lock);
1088  if (pipi != NULL) {
1089  if (pipi->type == LINUX_SMP_IPI &&
1090  (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
1091  IPIQ[cpu].resched_flag = 0;
1092  ipi_decode(pipi);
1093  }
1094  /*
1095  * The use of the __raw_local restore isn't
1096  * as obviously necessary here as in smtc_ipi_replay(),
1097  * but it's more efficient, given that we're already
1098  * running down the IPI queue.
1099  */
1100  __arch_local_irq_restore(flags);
1101  }
1102 }
1103 
1104 /*
1105  * Cross-VPE interrupts in the SMTC prototype use "software interrupts"
1106  * set via cross-VPE MTTR manipulation of the Cause register. It would be
1107  * in some regards preferable to have external logic for "doorbell" hardware
1108  * interrupts.
1109  */
1110 
1111 static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ;
1112 
1113 static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
1114 {
1115  int my_vpe = cpu_data[smp_processor_id()].vpe_id;
1116  int my_tc = cpu_data[smp_processor_id()].tc_id;
1117  int cpu;
1118  struct smtc_ipi *pipi;
1119  unsigned long tcstatus;
1120  int sent;
1121  unsigned long flags;
1122  unsigned int mtflags;
1123  unsigned int vpflags;
1124 
1125  /*
1126  * So long as cross-VPE interrupts are done via
1127  * MFTR/MTTR read-modify-writes of Cause, we need
1128  * to stop other VPEs whenever the local VPE does
1129  * anything similar.
1130  */
1131  local_irq_save(flags);
1132  vpflags = dvpe();
1133  clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ);
1134  set_c0_status(0x100 << MIPS_CPU_IPI_IRQ);
1135  irq_enable_hazard();
1136  evpe(vpflags);
1137  local_irq_restore(flags);
1138 
1139  /*
1140  * Cross-VPE Interrupt handler: Try to directly deliver IPIs
1141  * queued for TCs on this VPE other than the current one.
1142  * Return-from-interrupt should cause us to drain the queue
1143  * for the current TC, so we ought not to have to do it explicitly here.
1144  */
1145 
1146  for_each_online_cpu(cpu) {
1147  if (cpu_data[cpu].vpe_id != my_vpe)
1148  continue;
1149 
1150  pipi = smtc_ipi_dq(&IPIQ[cpu]);
1151  if (pipi != NULL) {
1152  if (cpu_data[cpu].tc_id != my_tc) {
1153  sent = 0;
1154  LOCK_MT_PRA();
1155  settc(cpu_data[cpu].tc_id);
1157  mips_ihb();
1158  tcstatus = read_tc_c0_tcstatus();
1159  if ((tcstatus & TCSTATUS_IXMT) == 0) {
1160  post_direct_ipi(cpu, pipi);
1161  sent = 1;
1162  }
1163  write_tc_c0_tchalt(0);
1164  UNLOCK_MT_PRA();
1165  if (!sent) {
1166  smtc_ipi_req(&IPIQ[cpu], pipi);
1167  }
1168  } else {
1169  /*
1170  * ipi_decode() should be called
1171  * with interrupts off
1172  */
1173  local_irq_save(flags);
1174  if (pipi->type == LINUX_SMP_IPI &&
1175  (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
1176  IPIQ[cpu].resched_flag = 0;
1177  ipi_decode(pipi);
1178  local_irq_restore(flags);
1179  }
1180  }
1181  }
1182 
1183  return IRQ_HANDLED;
1184 }
1185 
1186 static void ipi_irq_dispatch(void)
1187 {
1188  do_IRQ(cpu_ipi_irq);
1189 }
1190 
1191 static struct irqaction irq_ipi = {
1192  .handler = ipi_interrupt,
1193  .flags = IRQF_PERCPU,
1194  .name = "SMTC_IPI"
1195 };
1196 
1197 static void setup_cross_vpe_interrupts(unsigned int nvpe)
1198 {
1199  if (nvpe < 1)
1200  return;
1201 
1202  if (!cpu_has_vint)
1203  panic("SMTC Kernel requires Vectored Interrupt support");
1204 
1205  set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch);
1206 
1207  setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ));
1208 
1209  irq_set_handler(cpu_ipi_irq, handle_percpu_irq);
1210 }
1211 
1212 /*
1213  * SMTC-specific hacks invoked from elsewhere in the kernel.
1214  */
1215 
1216  /*
1217  * smtc_ipi_replay is called from raw_local_irq_restore
1218  */
1219 
1221 {
1222  unsigned int cpu = smp_processor_id();
1223 
1224  /*
1225  * To the extent that we've ever turned interrupts off,
1226  * we may have accumulated deferred IPIs. This is subtle.
1227  * we should be OK: If we pick up something and dispatch
1228  * it here, that's great. If we see nothing, but concurrent
1229  * with this operation, another TC sends us an IPI, IXMT
1230  * is clear, and we'll handle it as a real pseudo-interrupt
1231  * and not a pseudo-pseudo interrupt. The important thing
1232  * is to do the last check for queued message *after* the
1233  * re-enabling of interrupts.
1234  */
1235  while (IPIQ[cpu].head != NULL) {
1236  struct smtc_ipi_q *q = &IPIQ[cpu];
1237  struct smtc_ipi *pipi;
1238  unsigned long flags;
1239 
1240  /*
1241  * It's just possible we'll come in with interrupts
1242  * already enabled.
1243  */
1244  local_irq_save(flags);
1245 
1246  spin_lock(&q->lock);
1247  pipi = __smtc_ipi_dq(q);
1248  spin_unlock(&q->lock);
1249  /*
1250  ** But use a raw restore here to avoid recursion.
1251  */
1252  __arch_local_irq_restore(flags);
1253 
1254  if (pipi) {
1255  self_ipi(pipi);
1256  smtc_cpu_stats[cpu].selfipis++;
1257  }
1258  }
1259 }
1260 
1262 
1264 {
1265 #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
1266  int im;
1267  int flags;
1268  int mtflags;
1269  int bit;
1270  int vpe;
1271  int tc;
1272  int hook_ntcs;
1273  /*
1274  * printk within DMT-protected regions can deadlock,
1275  * so buffer diagnostic messages for later output.
1276  */
1277  char *pdb_msg;
1278  char id_ho_db_msg[768]; /* worst-case use should be less than 700 */
1279 
1280  if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */
1281  if (atomic_add_return(1, &idle_hook_initialized) == 1) {
1282  int mvpconf0;
1283  /* Tedious stuff to just do once */
1284  mvpconf0 = read_c0_mvpconf0();
1285  hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
1286  if (hook_ntcs > NR_CPUS)
1287  hook_ntcs = NR_CPUS;
1288  for (tc = 0; tc < hook_ntcs; tc++) {
1289  tcnoprog[tc] = 0;
1290  clock_hang_reported[tc] = 0;
1291  }
1292  for (vpe = 0; vpe < 2; vpe++)
1293  for (im = 0; im < 8; im++)
1294  imstuckcount[vpe][im] = 0;
1295  printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs);
1296  atomic_set(&idle_hook_initialized, 1000);
1297  } else {
1298  /* Someone else is initializing in parallel - let 'em finish */
1299  while (atomic_read(&idle_hook_initialized) < 1000)
1300  ;
1301  }
1302  }
1303 
1304  /* Have we stupidly left IXMT set somewhere? */
1305  if (read_c0_tcstatus() & 0x400) {
1306  write_c0_tcstatus(read_c0_tcstatus() & ~0x400);
1307  ehb();
1308  printk("Dangling IXMT in cpu_idle()\n");
1309  }
1310 
1311  /* Have we stupidly left an IM bit turned off? */
1312 #define IM_LIMIT 2000
1313  local_irq_save(flags);
1314  mtflags = dmt();
1315  pdb_msg = &id_ho_db_msg[0];
1316  im = read_c0_status();
1317  vpe = current_cpu_data.vpe_id;
1318  for (bit = 0; bit < 8; bit++) {
1319  /*
1320  * In current prototype, I/O interrupts
1321  * are masked for VPE > 0
1322  */
1323  if (vpemask[vpe][bit]) {
1324  if (!(im & (0x100 << bit)))
1325  imstuckcount[vpe][bit]++;
1326  else
1327  imstuckcount[vpe][bit] = 0;
1328  if (imstuckcount[vpe][bit] > IM_LIMIT) {
1329  set_c0_status(0x100 << bit);
1330  ehb();
1331  imstuckcount[vpe][bit] = 0;
1332  pdb_msg += sprintf(pdb_msg,
1333  "Dangling IM %d fixed for VPE %d\n", bit,
1334  vpe);
1335  }
1336  }
1337  }
1338 
1339  emt(mtflags);
1340  local_irq_restore(flags);
1341  if (pdb_msg != &id_ho_db_msg[0])
1342  printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg);
1343 #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
1344 
1345  smtc_ipi_replay();
1346 }
1347 
1348 void smtc_soft_dump(void)
1349 {
1350  int i;
1351 
1352  printk("Counter Interrupts taken per CPU (TC)\n");
1353  for (i=0; i < NR_CPUS; i++) {
1354  printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints);
1355  }
1356  printk("Self-IPI invocations:\n");
1357  for (i=0; i < NR_CPUS; i++) {
1358  printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis);
1359  }
1360  smtc_ipi_qdump();
1361  printk("%d Recoveries of \"stolen\" FPU\n",
1363 }
1364 
1365 
1366 /*
1367  * TLB management routines special to SMTC
1368  */
1369 
1370 void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu)
1371 {
1372  unsigned long flags, mtflags, tcstat, prevhalt, asid;
1373  int tlb, i;
1374 
1375  /*
1376  * It would be nice to be able to use a spinlock here,
1377  * but this is invoked from within TLB flush routines
1378  * that protect themselves with DVPE, so if a lock is
1379  * held by another TC, it'll never be freed.
1380  *
1381  * DVPE/DMT must not be done with interrupts enabled,
1382  * so even so most callers will already have disabled
1383  * them, let's be really careful...
1384  */
1385 
1386  local_irq_save(flags);
1387  if (smtc_status & SMTC_TLB_SHARED) {
1388  mtflags = dvpe();
1389  tlb = 0;
1390  } else {
1391  mtflags = dmt();
1392  tlb = cpu_data[cpu].vpe_id;
1393  }
1394  asid = asid_cache(cpu);
1395 
1396  do {
1397  if (!((asid += ASID_INC) & ASID_MASK) ) {
1398  if (cpu_has_vtag_icache)
1399  flush_icache_all();
1400  /* Traverse all online CPUs (hack requires contiguous range) */
1401  for_each_online_cpu(i) {
1402  /*
1403  * We don't need to worry about our own CPU, nor those of
1404  * CPUs who don't share our TLB.
1405  */
1406  if ((i != smp_processor_id()) &&
1407  ((smtc_status & SMTC_TLB_SHARED) ||
1408  (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) {
1409  settc(cpu_data[i].tc_id);
1410  prevhalt = read_tc_c0_tchalt() & TCHALT_H;
1411  if (!prevhalt) {
1413  mips_ihb();
1414  }
1415  tcstat = read_tc_c0_tcstatus();
1416  smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i);
1417  if (!prevhalt)
1418  write_tc_c0_tchalt(0);
1419  }
1420  }
1421  if (!asid) /* fix version if needed */
1422  asid = ASID_FIRST_VERSION;
1423  local_flush_tlb_all(); /* start new asid cycle */
1424  }
1425  } while (smtc_live_asid[tlb][(asid & ASID_MASK)]);
1426 
1427  /*
1428  * SMTC shares the TLB within VPEs and possibly across all VPEs.
1429  */
1430  for_each_online_cpu(i) {
1431  if ((smtc_status & SMTC_TLB_SHARED) ||
1432  (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))
1433  cpu_context(i, mm) = asid_cache(i) = asid;
1434  }
1435 
1436  if (smtc_status & SMTC_TLB_SHARED)
1437  evpe(mtflags);
1438  else
1439  emt(mtflags);
1440  local_irq_restore(flags);
1441 }
1442 
1443 /*
1444  * Invoked from macros defined in mmu_context.h
1445  * which must already have disabled interrupts
1446  * and done a DVPE or DMT as appropriate.
1447  */
1448 
1449 void smtc_flush_tlb_asid(unsigned long asid)
1450 {
1451  int entry;
1452  unsigned long ehi;
1453 
1454  entry = read_c0_wired();
1455 
1456  /* Traverse all non-wired entries */
1457  while (entry < current_cpu_data.tlbsize) {
1458  write_c0_index(entry);
1459  ehb();
1460  tlb_read();
1461  ehb();
1462  ehi = read_c0_entryhi();
1463  if ((ehi & ASID_MASK) == asid) {
1464  /*
1465  * Invalidate only entries with specified ASID,
1466  * makiing sure all entries differ.
1467  */
1468  write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
1469  write_c0_entrylo0(0);
1470  write_c0_entrylo1(0);
1471  mtc0_tlbw_hazard();
1472  tlb_write_indexed();
1473  }
1474  entry++;
1475  }
1477  tlbw_use_hazard();
1478 }
1479 
1480 /*
1481  * Support for single-threading cache flush operations.
1482  */
1483 
1484 static int halt_state_save[NR_CPUS];
1485 
1486 /*
1487  * To really, really be sure that nothing is being done
1488  * by other TCs, halt them all. This code assumes that
1489  * a DVPE has already been done, so while their Halted
1490  * state is theoretically architecturally unstable, in
1491  * practice, it's not going to change while we're looking
1492  * at it.
1493  */
1494 
1496 {
1497  int cpu;
1498 
1499  for_each_online_cpu(cpu) {
1500  if (cpu != smp_processor_id()) {
1501  settc(cpu_data[cpu].tc_id);
1502  halt_state_save[cpu] = read_tc_c0_tchalt();
1504  }
1505  }
1506  mips_ihb();
1507 }
1508 
1509 /* It would be cheating to change the cpu_online states during a flush! */
1510 
1512 {
1513  int cpu;
1514 
1515  /*
1516  * Start with a hazard barrier to ensure
1517  * that all CACHE ops have played through.
1518  */
1519  mips_ihb();
1520 
1521  for_each_online_cpu(cpu) {
1522  if (cpu != smp_processor_id()) {
1523  settc(cpu_data[cpu].tc_id);
1524  write_tc_c0_tchalt(halt_state_save[cpu]);
1525  }
1526  }
1527  mips_ihb();
1528 }