Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
beat_htab.c
Go to the documentation of this file.
1 /*
2  * "Cell Reference Set" HTAB support.
3  *
4  * (C) Copyright 2006-2007 TOSHIBA CORPORATION
5  *
6  * This code is based on arch/powerpc/platforms/pseries/lpar.c:
7  * Copyright (C) 2001 Todd Inglett, IBM Corporation
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License along
20  * with this program; if not, write to the Free Software Foundation, Inc.,
21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22  */
23 
24 #undef DEBUG_LOW
25 
26 #include <linux/kernel.h>
27 #include <linux/spinlock.h>
28 
29 #include <asm/mmu.h>
30 #include <asm/page.h>
31 #include <asm/pgtable.h>
32 #include <asm/machdep.h>
33 #include <asm/udbg.h>
34 
35 #include "beat_wrapper.h"
36 
37 #ifdef DEBUG_LOW
38 #define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
39 #else
40 #define DBG_LOW(fmt...) do { } while (0)
41 #endif
42 
43 static DEFINE_RAW_SPINLOCK(beat_htab_lock);
44 
45 static inline unsigned int beat_read_mask(unsigned hpte_group)
46 {
47  unsigned long rmask = 0;
48  u64 hpte_v[5];
49 
50  beat_read_htab_entries(0, hpte_group + 0, hpte_v);
51  if (!(hpte_v[0] & HPTE_V_BOLTED))
52  rmask |= 0x8000;
53  if (!(hpte_v[1] & HPTE_V_BOLTED))
54  rmask |= 0x4000;
55  if (!(hpte_v[2] & HPTE_V_BOLTED))
56  rmask |= 0x2000;
57  if (!(hpte_v[3] & HPTE_V_BOLTED))
58  rmask |= 0x1000;
59  beat_read_htab_entries(0, hpte_group + 4, hpte_v);
60  if (!(hpte_v[0] & HPTE_V_BOLTED))
61  rmask |= 0x0800;
62  if (!(hpte_v[1] & HPTE_V_BOLTED))
63  rmask |= 0x0400;
64  if (!(hpte_v[2] & HPTE_V_BOLTED))
65  rmask |= 0x0200;
66  if (!(hpte_v[3] & HPTE_V_BOLTED))
67  rmask |= 0x0100;
68  hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP);
69  beat_read_htab_entries(0, hpte_group + 0, hpte_v);
70  if (!(hpte_v[0] & HPTE_V_BOLTED))
71  rmask |= 0x80;
72  if (!(hpte_v[1] & HPTE_V_BOLTED))
73  rmask |= 0x40;
74  if (!(hpte_v[2] & HPTE_V_BOLTED))
75  rmask |= 0x20;
76  if (!(hpte_v[3] & HPTE_V_BOLTED))
77  rmask |= 0x10;
78  beat_read_htab_entries(0, hpte_group + 4, hpte_v);
79  if (!(hpte_v[0] & HPTE_V_BOLTED))
80  rmask |= 0x08;
81  if (!(hpte_v[1] & HPTE_V_BOLTED))
82  rmask |= 0x04;
83  if (!(hpte_v[2] & HPTE_V_BOLTED))
84  rmask |= 0x02;
85  if (!(hpte_v[3] & HPTE_V_BOLTED))
86  rmask |= 0x01;
87  return rmask;
88 }
89 
90 static long beat_lpar_hpte_insert(unsigned long hpte_group,
91  unsigned long vpn, unsigned long pa,
92  unsigned long rflags, unsigned long vflags,
93  int psize, int ssize)
94 {
95  unsigned long lpar_rc;
96  u64 hpte_v, hpte_r, slot;
97 
98  if (vflags & HPTE_V_SECONDARY)
99  return -1;
100 
101  if (!(vflags & HPTE_V_BOLTED))
102  DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
103  "rflags=%lx, vflags=%lx, psize=%d)\n",
104  hpte_group, va, pa, rflags, vflags, psize);
105 
106  hpte_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M) |
107  vflags | HPTE_V_VALID;
108  hpte_r = hpte_encode_r(pa, psize) | rflags;
109 
110  if (!(vflags & HPTE_V_BOLTED))
111  DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
112 
113  if (rflags & _PAGE_NO_CACHE)
114  hpte_r &= ~_PAGE_COHERENT;
115 
116  raw_spin_lock(&beat_htab_lock);
117  lpar_rc = beat_read_mask(hpte_group);
118  if (lpar_rc == 0) {
119  if (!(vflags & HPTE_V_BOLTED))
120  DBG_LOW(" full\n");
121  raw_spin_unlock(&beat_htab_lock);
122  return -1;
123  }
124 
125  lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
126  hpte_v, hpte_r, &slot);
127  raw_spin_unlock(&beat_htab_lock);
128 
129  /*
130  * Since we try and ioremap PHBs we don't own, the pte insert
131  * will fail. However we must catch the failure in hash_page
132  * or we will loop forever, so return -2 in this case.
133  */
134  if (unlikely(lpar_rc != 0)) {
135  if (!(vflags & HPTE_V_BOLTED))
136  DBG_LOW(" lpar err %lx\n", lpar_rc);
137  return -2;
138  }
139  if (!(vflags & HPTE_V_BOLTED))
140  DBG_LOW(" -> slot: %lx\n", slot);
141 
142  /* We have to pass down the secondary bucket bit here as well */
143  return (slot ^ hpte_group) & 15;
144 }
145 
146 static long beat_lpar_hpte_remove(unsigned long hpte_group)
147 {
148  DBG_LOW("hpte_remove(group=%lx)\n", hpte_group);
149  return -1;
150 }
151 
152 static unsigned long beat_lpar_hpte_getword0(unsigned long slot)
153 {
154  unsigned long dword0;
155  unsigned long lpar_rc;
156  u64 dword[5];
157 
158  lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword);
159 
160  dword0 = dword[slot&3];
161 
162  BUG_ON(lpar_rc != 0);
163 
164  return dword0;
165 }
166 
167 static void beat_lpar_hptab_clear(void)
168 {
169  unsigned long size_bytes = 1UL << ppc64_pft_size;
170  unsigned long hpte_count = size_bytes >> 4;
171  int i;
172  u64 dummy0, dummy1;
173 
174  /* TODO: Use bulk call */
175  for (i = 0; i < hpte_count; i++)
176  beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1);
177 }
178 
179 /*
180  * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
181  * the low 3 bits of flags happen to line up. So no transform is needed.
182  * We can probably optimize here and assume the high bits of newpp are
183  * already zero. For now I am paranoid.
184  */
185 static long beat_lpar_hpte_updatepp(unsigned long slot,
186  unsigned long newpp,
187  unsigned long vpn,
188  int psize, int ssize, int local)
189 {
190  unsigned long lpar_rc;
191  u64 dummy0, dummy1;
192  unsigned long want_v;
193 
194  want_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M);
195 
196  DBG_LOW(" update: "
197  "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
198  want_v & HPTE_V_AVPN, slot, psize, newpp);
199 
200  raw_spin_lock(&beat_htab_lock);
201  dummy0 = beat_lpar_hpte_getword0(slot);
202  if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
203  DBG_LOW("not found !\n");
204  raw_spin_unlock(&beat_htab_lock);
205  return -1;
206  }
207 
208  lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
209  &dummy1);
210  raw_spin_unlock(&beat_htab_lock);
211  if (lpar_rc != 0 || dummy0 == 0) {
212  DBG_LOW("not found !\n");
213  return -1;
214  }
215 
216  DBG_LOW("ok %lx %lx\n", dummy0, dummy1);
217 
218  BUG_ON(lpar_rc != 0);
219 
220  return 0;
221 }
222 
223 static long beat_lpar_hpte_find(unsigned long vpn, int psize)
224 {
225  unsigned long hash;
226  unsigned long i, j;
227  long slot;
228  unsigned long want_v, hpte_v;
229 
230  hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
231  want_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M);
232 
233  for (j = 0; j < 2; j++) {
234  slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
235  for (i = 0; i < HPTES_PER_GROUP; i++) {
236  hpte_v = beat_lpar_hpte_getword0(slot);
237 
238  if (HPTE_V_COMPARE(hpte_v, want_v)
239  && (hpte_v & HPTE_V_VALID)
240  && (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
241  /* HPTE matches */
242  if (j)
243  slot = -slot;
244  return slot;
245  }
246  ++slot;
247  }
248  hash = ~hash;
249  }
250 
251  return -1;
252 }
253 
254 static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
255  unsigned long ea,
256  int psize, int ssize)
257 {
258  unsigned long vpn;
259  unsigned long lpar_rc, slot, vsid;
260  u64 dummy0, dummy1;
261 
262  vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
263  vpn = hpt_vpn(ea, vsid, MMU_SEGSIZE_256M);
264 
265  raw_spin_lock(&beat_htab_lock);
266  slot = beat_lpar_hpte_find(vpn, psize);
267  BUG_ON(slot == -1);
268 
269  lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7,
270  &dummy0, &dummy1);
271  raw_spin_unlock(&beat_htab_lock);
272 
273  BUG_ON(lpar_rc != 0);
274 }
275 
276 static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
277  int psize, int ssize, int local)
278 {
279  unsigned long want_v;
280  unsigned long lpar_rc;
281  u64 dummy1, dummy2;
282  unsigned long flags;
283 
284  DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
285  slot, va, psize, local);
286  want_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M);
287 
288  raw_spin_lock_irqsave(&beat_htab_lock, flags);
289  dummy1 = beat_lpar_hpte_getword0(slot);
290 
291  if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
292  DBG_LOW("not found !\n");
293  raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
294  return;
295  }
296 
297  lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
298  &dummy1, &dummy2);
299  raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
300 
301  BUG_ON(lpar_rc != 0);
302 }
303 
304 void __init hpte_init_beat(void)
305 {
306  ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate;
307  ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp;
308  ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
309  ppc_md.hpte_insert = beat_lpar_hpte_insert;
310  ppc_md.hpte_remove = beat_lpar_hpte_remove;
311  ppc_md.hpte_clear_all = beat_lpar_hptab_clear;
312 }
313 
314 static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
315  unsigned long vpn, unsigned long pa,
316  unsigned long rflags, unsigned long vflags,
317  int psize, int ssize)
318 {
319  unsigned long lpar_rc;
320  u64 hpte_v, hpte_r, slot;
321 
322  if (vflags & HPTE_V_SECONDARY)
323  return -1;
324 
325  if (!(vflags & HPTE_V_BOLTED))
326  DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, "
327  "rflags=%lx, vflags=%lx, psize=%d)\n",
328  hpte_group, vpn, pa, rflags, vflags, psize);
329 
330  hpte_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M) |
331  vflags | HPTE_V_VALID;
332  hpte_r = hpte_encode_r(pa, psize) | rflags;
333 
334  if (!(vflags & HPTE_V_BOLTED))
335  DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
336 
337  if (rflags & _PAGE_NO_CACHE)
338  hpte_r &= ~_PAGE_COHERENT;
339 
340  /* insert into not-volted entry */
341  lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
342  HPTE_V_BOLTED, 0, &slot);
343  /*
344  * Since we try and ioremap PHBs we don't own, the pte insert
345  * will fail. However we must catch the failure in hash_page
346  * or we will loop forever, so return -2 in this case.
347  */
348  if (unlikely(lpar_rc != 0)) {
349  if (!(vflags & HPTE_V_BOLTED))
350  DBG_LOW(" lpar err %lx\n", lpar_rc);
351  return -2;
352  }
353  if (!(vflags & HPTE_V_BOLTED))
354  DBG_LOW(" -> slot: %lx\n", slot);
355 
356  /* We have to pass down the secondary bucket bit here as well */
357  return (slot ^ hpte_group) & 15;
358 }
359 
360 /*
361  * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
362  * the low 3 bits of flags happen to line up. So no transform is needed.
363  * We can probably optimize here and assume the high bits of newpp are
364  * already zero. For now I am paranoid.
365  */
366 static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
367  unsigned long newpp,
368  unsigned long vpn,
369  int psize, int ssize, int local)
370 {
371  unsigned long lpar_rc;
372  unsigned long want_v;
373  unsigned long pss;
374 
375  want_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M);
376  pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
377 
378  DBG_LOW(" update: "
379  "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
380  want_v & HPTE_V_AVPN, slot, psize, newpp);
381 
382  lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);
383 
384  if (lpar_rc == 0xfffffff7) {
385  DBG_LOW("not found !\n");
386  return -1;
387  }
388 
389  DBG_LOW("ok\n");
390 
391  BUG_ON(lpar_rc != 0);
392 
393  return 0;
394 }
395 
396 static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,
397  int psize, int ssize, int local)
398 {
399  unsigned long want_v;
400  unsigned long lpar_rc;
401  unsigned long pss;
402 
403  DBG_LOW(" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
404  slot, vpn, psize, local);
405  want_v = hpte_encode_v(vpn, psize, MMU_SEGSIZE_256M);
406  pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
407 
408  lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
409 
410  /* E_busy can be valid output: page may be already replaced */
411  BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
412 }
413 
414 static int64_t _beat_lpar_hptab_clear_v3(void)
415 {
416  return beat_clear_htab3(0);
417 }
418 
419 static void beat_lpar_hptab_clear_v3(void)
420 {
421  _beat_lpar_hptab_clear_v3();
422 }
423 
424 void __init hpte_init_beat_v3(void)
425 {
426  if (_beat_lpar_hptab_clear_v3() == 0) {
427  ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate_v3;
428  ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp_v3;
429  ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
430  ppc_md.hpte_insert = beat_lpar_hpte_insert_v3;
431  ppc_md.hpte_remove = beat_lpar_hpte_remove;
432  ppc_md.hpte_clear_all = beat_lpar_hptab_clear_v3;
433  } else {
434  ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate;
435  ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp;
436  ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
437  ppc_md.hpte_insert = beat_lpar_hpte_insert;
438  ppc_md.hpte_remove = beat_lpar_hpte_remove;
439  ppc_md.hpte_clear_all = beat_lpar_hptab_clear;
440  }
441 }
Generated on Thu Jan 10 2013 13:14:49 for Linux Kernel by   doxygen 1.8.2