Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
cvmx-bootmem.c
Go to the documentation of this file.
1 /***********************license start***************
2  * Author: Cavium Networks
3  *
4  * Contact: [email protected]
5  * This file is part of the OCTEON SDK
6  *
7  * Copyright (c) 2003-2008 Cavium Networks
8  *
9  * This file is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License, Version 2, as
11  * published by the Free Software Foundation.
12  *
13  * This file is distributed in the hope that it will be useful, but
14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16  * NONINFRINGEMENT. See the GNU General Public License for more
17  * details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this file; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22  * or visit http://www.gnu.org/licenses/.
23  *
24  * This file may also be available under a different license from Cavium.
25  * Contact Cavium Networks for more information
26  ***********************license end**************************************/
27 
28 /*
29  * Simple allocate only memory allocator. Used to allocate memory at
30  * application start time.
31  */
32 
33 #include <linux/kernel.h>
34 #include <linux/module.h>
35 
36 #include <asm/octeon/cvmx.h>
39 
40 /*#define DEBUG */
41 
42 
44 
45 /* See header file for descriptions of functions */
46 
47 /*
48  * Wrapper functions are provided for reading/writing the size and
49  * next block values as these may not be directly addressible (in 32
50  * bit applications, for instance.) Offsets of data elements in
51  * bootmem list, must match cvmx_bootmem_block_header_t.
52  */
53 #define NEXT_OFFSET 0
54 #define SIZE_OFFSET 8
55 
56 static void cvmx_bootmem_phy_set_size(uint64_t addr, uint64_t size)
57 {
58  cvmx_write64_uint64((addr + SIZE_OFFSET) | (1ull << 63), size);
59 }
60 
61 static void cvmx_bootmem_phy_set_next(uint64_t addr, uint64_t next)
62 {
63  cvmx_write64_uint64((addr + NEXT_OFFSET) | (1ull << 63), next);
64 }
65 
66 static uint64_t cvmx_bootmem_phy_get_size(uint64_t addr)
67 {
68  return cvmx_read64_uint64((addr + SIZE_OFFSET) | (1ull << 63));
69 }
70 
71 static uint64_t cvmx_bootmem_phy_get_next(uint64_t addr)
72 {
73  return cvmx_read64_uint64((addr + NEXT_OFFSET) | (1ull << 63));
74 }
75 
77  uint64_t min_addr, uint64_t max_addr)
78 {
79  int64_t address;
80  address =
81  cvmx_bootmem_phy_alloc(size, min_addr, max_addr, alignment, 0);
82 
83  if (address > 0)
84  return cvmx_phys_to_ptr(address);
85  else
86  return NULL;
87 }
88 
91 {
92  return cvmx_bootmem_alloc_range(size, alignment, address,
93  address + size);
94 }
95 
97 {
98  return cvmx_bootmem_alloc_range(size, alignment, 0, 0);
99 }
100 
102  uint64_t max_addr, uint64_t align,
103  char *name)
104 {
105  int64_t addr;
106 
107  addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
108  align, name, 0);
109  if (addr >= 0)
110  return cvmx_phys_to_ptr(addr);
111  else
112  return NULL;
113 }
114 
116  char *name)
117 {
118  return cvmx_bootmem_alloc_named_range(size, address, address + size,
119  0, name);
120 }
121 
123 {
124  return cvmx_bootmem_alloc_named_range(size, 0, 0, alignment, name);
125 }
127 
129 {
130  return cvmx_bootmem_phy_named_block_free(name, 0);
131 }
132 
134 {
135  return cvmx_bootmem_phy_named_block_find(name, 0);
136 }
138 
140 {
141  cvmx_spinlock_lock((cvmx_spinlock_t *) &(cvmx_bootmem_desc->lock));
142 }
143 
145 {
146  cvmx_spinlock_unlock((cvmx_spinlock_t *) &(cvmx_bootmem_desc->lock));
147 }
148 
149 int cvmx_bootmem_init(void *mem_desc_ptr)
150 {
151  /* Here we set the global pointer to the bootmem descriptor
152  * block. This pointer will be used directly, so we will set
153  * it up to be directly usable by the application. It is set
154  * up as follows for the various runtime/ABI combinations:
155  *
156  * Linux 64 bit: Set XKPHYS bit
157  * Linux 32 bit: use mmap to create mapping, use virtual address
158  * CVMX 64 bit: use physical address directly
159  * CVMX 32 bit: use physical address directly
160  *
161  * Note that the CVMX environment assumes the use of 1-1 TLB
162  * mappings so that the physical addresses can be used
163  * directly
164  */
165  if (!cvmx_bootmem_desc) {
166 #if defined(CVMX_ABI_64)
167  /* Set XKPHYS bit */
168  cvmx_bootmem_desc = cvmx_phys_to_ptr(CAST64(mem_desc_ptr));
169 #else
170  cvmx_bootmem_desc = (struct cvmx_bootmem_desc *) mem_desc_ptr;
171 #endif
172  }
173 
174  return 0;
175 }
176 
177 /*
178  * The cvmx_bootmem_phy* functions below return 64 bit physical
179  * addresses, and expose more features that the cvmx_bootmem_functions
180  * above. These are required for full memory space access in 32 bit
181  * applications, as well as for using some advance features. Most
182  * applications should not need to use these.
183  */
184 
185 int64_t cvmx_bootmem_phy_alloc(uint64_t req_size, uint64_t address_min,
186  uint64_t address_max, uint64_t alignment,
187  uint32_t flags)
188 {
189 
191  uint64_t ent_addr;
192  /* points to previous list entry, NULL current entry is head of list */
193  uint64_t prev_addr = 0;
194  uint64_t new_ent_addr = 0;
195  uint64_t desired_min_addr;
196 
197 #ifdef DEBUG
198  cvmx_dprintf("cvmx_bootmem_phy_alloc: req_size: 0x%llx, "
199  "min_addr: 0x%llx, max_addr: 0x%llx, align: 0x%llx\n",
200  (unsigned long long)req_size,
201  (unsigned long long)address_min,
202  (unsigned long long)address_max,
203  (unsigned long long)alignment);
204 #endif
205 
206  if (cvmx_bootmem_desc->major_version > 3) {
207  cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
208  "version: %d.%d at addr: %p\n",
209  (int)cvmx_bootmem_desc->major_version,
210  (int)cvmx_bootmem_desc->minor_version,
211  cvmx_bootmem_desc);
212  goto error_out;
213  }
214 
215  /*
216  * Do a variety of checks to validate the arguments. The
217  * allocator code will later assume that these checks have
218  * been made. We validate that the requested constraints are
219  * not self-contradictory before we look through the list of
220  * available memory.
221  */
222 
223  /* 0 is not a valid req_size for this allocator */
224  if (!req_size)
225  goto error_out;
226 
227  /* Round req_size up to mult of minimum alignment bytes */
228  req_size = (req_size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
230 
231  /*
232  * Convert !0 address_min and 0 address_max to special case of
233  * range that specifies an exact memory block to allocate. Do
234  * this before other checks and adjustments so that this
235  * tranformation will be validated.
236  */
237  if (address_min && !address_max)
238  address_max = address_min + req_size;
239  else if (!address_min && !address_max)
240  address_max = ~0ull; /* If no limits given, use max limits */
241 
242 
243  /*
244  * Enforce minimum alignment (this also keeps the minimum free block
245  * req_size the same as the alignment req_size.
246  */
247  if (alignment < CVMX_BOOTMEM_ALIGNMENT_SIZE)
248  alignment = CVMX_BOOTMEM_ALIGNMENT_SIZE;
249 
250  /*
251  * Adjust address minimum based on requested alignment (round
252  * up to meet alignment). Do this here so we can reject
253  * impossible requests up front. (NOP for address_min == 0)
254  */
255  if (alignment)
256  address_min = ALIGN(address_min, alignment);
257 
258  /*
259  * Reject inconsistent args. We have adjusted these, so this
260  * may fail due to our internal changes even if this check
261  * would pass for the values the user supplied.
262  */
263  if (req_size > address_max - address_min)
264  goto error_out;
265 
266  /* Walk through the list entries - first fit found is returned */
267 
268  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
270  head_addr = cvmx_bootmem_desc->head_addr;
271  ent_addr = head_addr;
272  for (; ent_addr;
273  prev_addr = ent_addr,
274  ent_addr = cvmx_bootmem_phy_get_next(ent_addr)) {
275  uint64_t usable_base, usable_max;
276  uint64_t ent_size = cvmx_bootmem_phy_get_size(ent_addr);
277 
278  if (cvmx_bootmem_phy_get_next(ent_addr)
279  && ent_addr > cvmx_bootmem_phy_get_next(ent_addr)) {
280  cvmx_dprintf("Internal bootmem_alloc() error: ent: "
281  "0x%llx, next: 0x%llx\n",
282  (unsigned long long)ent_addr,
283  (unsigned long long)
284  cvmx_bootmem_phy_get_next(ent_addr));
285  goto error_out;
286  }
287 
288  /*
289  * Determine if this is an entry that can satisify the
290  * request Check to make sure entry is large enough to
291  * satisfy request.
292  */
293  usable_base =
294  ALIGN(max(address_min, ent_addr), alignment);
295  usable_max = min(address_max, ent_addr + ent_size);
296  /*
297  * We should be able to allocate block at address
298  * usable_base.
299  */
300 
301  desired_min_addr = usable_base;
302  /*
303  * Determine if request can be satisfied from the
304  * current entry.
305  */
306  if (!((ent_addr + ent_size) > usable_base
307  && ent_addr < address_max
308  && req_size <= usable_max - usable_base))
309  continue;
310  /*
311  * We have found an entry that has room to satisfy the
312  * request, so allocate it from this entry. If end
313  * CVMX_BOOTMEM_FLAG_END_ALLOC set, then allocate from
314  * the end of this block rather than the beginning.
315  */
316  if (flags & CVMX_BOOTMEM_FLAG_END_ALLOC) {
317  desired_min_addr = usable_max - req_size;
318  /*
319  * Align desired address down to required
320  * alignment.
321  */
322  desired_min_addr &= ~(alignment - 1);
323  }
324 
325  /* Match at start of entry */
326  if (desired_min_addr == ent_addr) {
327  if (req_size < ent_size) {
328  /*
329  * big enough to create a new block
330  * from top portion of block.
331  */
332  new_ent_addr = ent_addr + req_size;
333  cvmx_bootmem_phy_set_next(new_ent_addr,
334  cvmx_bootmem_phy_get_next(ent_addr));
335  cvmx_bootmem_phy_set_size(new_ent_addr,
336  ent_size -
337  req_size);
338 
339  /*
340  * Adjust next pointer as following
341  * code uses this.
342  */
343  cvmx_bootmem_phy_set_next(ent_addr,
344  new_ent_addr);
345  }
346 
347  /*
348  * adjust prev ptr or head to remove this
349  * entry from list.
350  */
351  if (prev_addr)
352  cvmx_bootmem_phy_set_next(prev_addr,
353  cvmx_bootmem_phy_get_next(ent_addr));
354  else
355  /*
356  * head of list being returned, so
357  * update head ptr.
358  */
359  cvmx_bootmem_desc->head_addr =
360  cvmx_bootmem_phy_get_next(ent_addr);
361 
362  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
364  return desired_min_addr;
365  }
366  /*
367  * block returned doesn't start at beginning of entry,
368  * so we know that we will be splitting a block off
369  * the front of this one. Create a new block from the
370  * beginning, add to list, and go to top of loop
371  * again.
372  *
373  * create new block from high portion of
374  * block, so that top block starts at desired
375  * addr.
376  */
377  new_ent_addr = desired_min_addr;
378  cvmx_bootmem_phy_set_next(new_ent_addr,
379  cvmx_bootmem_phy_get_next
380  (ent_addr));
381  cvmx_bootmem_phy_set_size(new_ent_addr,
382  cvmx_bootmem_phy_get_size
383  (ent_addr) -
384  (desired_min_addr -
385  ent_addr));
386  cvmx_bootmem_phy_set_size(ent_addr,
387  desired_min_addr - ent_addr);
388  cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
389  /* Loop again to handle actual alloc from new block */
390  }
391 error_out:
392  /* We didn't find anything, so return error */
393  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
395  return -1;
396 }
397 
399 {
400  uint64_t cur_addr;
401  uint64_t prev_addr = 0; /* zero is invalid */
402  int retval = 0;
403 
404 #ifdef DEBUG
405  cvmx_dprintf("__cvmx_bootmem_phy_free addr: 0x%llx, size: 0x%llx\n",
406  (unsigned long long)phy_addr, (unsigned long long)size);
407 #endif
408  if (cvmx_bootmem_desc->major_version > 3) {
409  cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
410  "version: %d.%d at addr: %p\n",
411  (int)cvmx_bootmem_desc->major_version,
412  (int)cvmx_bootmem_desc->minor_version,
413  cvmx_bootmem_desc);
414  return 0;
415  }
416 
417  /* 0 is not a valid size for this allocator */
418  if (!size)
419  return 0;
420 
421  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
423  cur_addr = cvmx_bootmem_desc->head_addr;
424  if (cur_addr == 0 || phy_addr < cur_addr) {
425  /* add at front of list - special case with changing head ptr */
426  if (cur_addr && phy_addr + size > cur_addr)
427  goto bootmem_free_done; /* error, overlapping section */
428  else if (phy_addr + size == cur_addr) {
429  /* Add to front of existing first block */
430  cvmx_bootmem_phy_set_next(phy_addr,
431  cvmx_bootmem_phy_get_next
432  (cur_addr));
433  cvmx_bootmem_phy_set_size(phy_addr,
434  cvmx_bootmem_phy_get_size
435  (cur_addr) + size);
436  cvmx_bootmem_desc->head_addr = phy_addr;
437 
438  } else {
439  /* New block before first block. OK if cur_addr is 0 */
440  cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
441  cvmx_bootmem_phy_set_size(phy_addr, size);
442  cvmx_bootmem_desc->head_addr = phy_addr;
443  }
444  retval = 1;
445  goto bootmem_free_done;
446  }
447 
448  /* Find place in list to add block */
449  while (cur_addr && phy_addr > cur_addr) {
450  prev_addr = cur_addr;
451  cur_addr = cvmx_bootmem_phy_get_next(cur_addr);
452  }
453 
454  if (!cur_addr) {
455  /*
456  * We have reached the end of the list, add on to end,
457  * checking to see if we need to combine with last
458  * block
459  */
460  if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) ==
461  phy_addr) {
462  cvmx_bootmem_phy_set_size(prev_addr,
463  cvmx_bootmem_phy_get_size
464  (prev_addr) + size);
465  } else {
466  cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
467  cvmx_bootmem_phy_set_size(phy_addr, size);
468  cvmx_bootmem_phy_set_next(phy_addr, 0);
469  }
470  retval = 1;
471  goto bootmem_free_done;
472  } else {
473  /*
474  * insert between prev and cur nodes, checking for
475  * merge with either/both.
476  */
477  if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) ==
478  phy_addr) {
479  /* Merge with previous */
480  cvmx_bootmem_phy_set_size(prev_addr,
481  cvmx_bootmem_phy_get_size
482  (prev_addr) + size);
483  if (phy_addr + size == cur_addr) {
484  /* Also merge with current */
485  cvmx_bootmem_phy_set_size(prev_addr,
486  cvmx_bootmem_phy_get_size(cur_addr) +
487  cvmx_bootmem_phy_get_size(prev_addr));
488  cvmx_bootmem_phy_set_next(prev_addr,
489  cvmx_bootmem_phy_get_next(cur_addr));
490  }
491  retval = 1;
492  goto bootmem_free_done;
493  } else if (phy_addr + size == cur_addr) {
494  /* Merge with current */
495  cvmx_bootmem_phy_set_size(phy_addr,
496  cvmx_bootmem_phy_get_size
497  (cur_addr) + size);
498  cvmx_bootmem_phy_set_next(phy_addr,
499  cvmx_bootmem_phy_get_next
500  (cur_addr));
501  cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
502  retval = 1;
503  goto bootmem_free_done;
504  }
505 
506  /* It is a standalone block, add in between prev and cur */
507  cvmx_bootmem_phy_set_size(phy_addr, size);
508  cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
509  cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
510 
511  }
512  retval = 1;
513 
514 bootmem_free_done:
515  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
517  return retval;
518 
519 }
520 
523 {
524  unsigned int i;
525  struct cvmx_bootmem_named_block_desc *named_block_array_ptr;
526 
527 #ifdef DEBUG
528  cvmx_dprintf("cvmx_bootmem_phy_named_block_find: %s\n", name);
529 #endif
530  /*
531  * Lock the structure to make sure that it is not being
532  * changed while we are examining it.
533  */
534  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
536 
537  /* Use XKPHYS for 64 bit linux */
538  named_block_array_ptr = (struct cvmx_bootmem_named_block_desc *)
539  cvmx_phys_to_ptr(cvmx_bootmem_desc->named_block_array_addr);
540 
541 #ifdef DEBUG
543  ("cvmx_bootmem_phy_named_block_find: named_block_array_ptr: %p\n",
544  named_block_array_ptr);
545 #endif
546  if (cvmx_bootmem_desc->major_version == 3) {
547  for (i = 0;
548  i < cvmx_bootmem_desc->named_block_num_blocks; i++) {
549  if ((name && named_block_array_ptr[i].size
550  && !strncmp(name, named_block_array_ptr[i].name,
551  cvmx_bootmem_desc->named_block_name_len
552  - 1))
553  || (!name && !named_block_array_ptr[i].size)) {
554  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
556 
557  return &(named_block_array_ptr[i]);
558  }
559  }
560  } else {
561  cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
562  "version: %d.%d at addr: %p\n",
563  (int)cvmx_bootmem_desc->major_version,
564  (int)cvmx_bootmem_desc->minor_version,
565  cvmx_bootmem_desc);
566  }
567  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
569 
570  return NULL;
571 }
572 
574 {
575  struct cvmx_bootmem_named_block_desc *named_block_ptr;
576 
577  if (cvmx_bootmem_desc->major_version != 3) {
578  cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: "
579  "%d.%d at addr: %p\n",
580  (int)cvmx_bootmem_desc->major_version,
581  (int)cvmx_bootmem_desc->minor_version,
582  cvmx_bootmem_desc);
583  return 0;
584  }
585 #ifdef DEBUG
586  cvmx_dprintf("cvmx_bootmem_phy_named_block_free: %s\n", name);
587 #endif
588 
589  /*
590  * Take lock here, as name lookup/block free/name free need to
591  * be atomic.
592  */
594 
595  named_block_ptr =
598  if (named_block_ptr) {
599 #ifdef DEBUG
600  cvmx_dprintf("cvmx_bootmem_phy_named_block_free: "
601  "%s, base: 0x%llx, size: 0x%llx\n",
602  name,
603  (unsigned long long)named_block_ptr->base_addr,
604  (unsigned long long)named_block_ptr->size);
605 #endif
606  __cvmx_bootmem_phy_free(named_block_ptr->base_addr,
607  named_block_ptr->size,
609  named_block_ptr->size = 0;
610  /* Set size to zero to indicate block not used. */
611  }
612 
614  return named_block_ptr != NULL; /* 0 on failure, 1 on success */
615 }
616 
618  uint64_t max_addr,
620  char *name,
621  uint32_t flags)
622 {
623  int64_t addr_allocated;
624  struct cvmx_bootmem_named_block_desc *named_block_desc_ptr;
625 
626 #ifdef DEBUG
627  cvmx_dprintf("cvmx_bootmem_phy_named_block_alloc: size: 0x%llx, min: "
628  "0x%llx, max: 0x%llx, align: 0x%llx, name: %s\n",
629  (unsigned long long)size,
630  (unsigned long long)min_addr,
631  (unsigned long long)max_addr,
632  (unsigned long long)alignment,
633  name);
634 #endif
635  if (cvmx_bootmem_desc->major_version != 3) {
636  cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: "
637  "%d.%d at addr: %p\n",
638  (int)cvmx_bootmem_desc->major_version,
639  (int)cvmx_bootmem_desc->minor_version,
640  cvmx_bootmem_desc);
641  return -1;
642  }
643 
644  /*
645  * Take lock here, as name lookup/block alloc/name add need to
646  * be atomic.
647  */
648  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
649  cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
650 
651  /* Get pointer to first available named block descriptor */
652  named_block_desc_ptr =
654  flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
655 
656  /*
657  * Check to see if name already in use, return error if name
658  * not available or no more room for blocks.
659  */
661  flags | CVMX_BOOTMEM_FLAG_NO_LOCKING) || !named_block_desc_ptr) {
662  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
663  cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
664  return -1;
665  }
666 
667 
668  /*
669  * Round size up to mult of minimum alignment bytes We need
670  * the actual size allocated to allow for blocks to be
671  * coallesced when they are freed. The alloc routine does the
672  * same rounding up on all allocations.
673  */
674  size = ALIGN(size, CVMX_BOOTMEM_ALIGNMENT_SIZE);
675 
676  addr_allocated = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
677  alignment,
678  flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
679  if (addr_allocated >= 0) {
680  named_block_desc_ptr->base_addr = addr_allocated;
681  named_block_desc_ptr->size = size;
682  strncpy(named_block_desc_ptr->name, name,
683  cvmx_bootmem_desc->named_block_name_len);
684  named_block_desc_ptr->name[cvmx_bootmem_desc->named_block_name_len - 1] = 0;
685  }
686 
687  if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
688  cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
689  return addr_allocated;
690 }