Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
icm.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
3  * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses. You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  * Redistribution and use in source and binary forms, with or
12  * without modification, are permitted provided that the following
13  * conditions are met:
14  *
15  * - Redistributions of source code must retain the above
16  * copyright notice, this list of conditions and the following
17  * disclaimer.
18  *
19  * - Redistributions in binary form must reproduce the above
20  * copyright notice, this list of conditions and the following
21  * disclaimer in the documentation and/or other materials
22  * provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/errno.h>
35 #include <linux/mm.h>
36 #include <linux/scatterlist.h>
37 #include <linux/slab.h>
38 
39 #include <linux/mlx4/cmd.h>
40 
41 #include "mlx4.h"
42 #include "icm.h"
43 #include "fw.h"
44 
45 /*
46  * We allocate in as big chunks as we can, up to a maximum of 256 KB
47  * per chunk.
48  */
49 enum {
52 };
53 
54 static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)
55 {
56  int i;
57 
58  if (chunk->nsg > 0)
59  pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,
61 
62  for (i = 0; i < chunk->npages; ++i)
63  __free_pages(sg_page(&chunk->mem[i]),
64  get_order(chunk->mem[i].length));
65 }
66 
67 static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)
68 {
69  int i;
70 
71  for (i = 0; i < chunk->npages; ++i)
72  dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length,
73  lowmem_page_address(sg_page(&chunk->mem[i])),
74  sg_dma_address(&chunk->mem[i]));
75 }
76 
77 void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent)
78 {
79  struct mlx4_icm_chunk *chunk, *tmp;
80 
81  if (!icm)
82  return;
83 
84  list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {
85  if (coherent)
86  mlx4_free_icm_coherent(dev, chunk);
87  else
88  mlx4_free_icm_pages(dev, chunk);
89 
90  kfree(chunk);
91  }
92 
93  kfree(icm);
94 }
95 
96 static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask)
97 {
98  struct page *page;
99 
100  page = alloc_pages(gfp_mask, order);
101  if (!page)
102  return -ENOMEM;
103 
104  sg_set_page(mem, page, PAGE_SIZE << order, 0);
105  return 0;
106 }
107 
108 static int mlx4_alloc_icm_coherent(struct device *dev, struct scatterlist *mem,
109  int order, gfp_t gfp_mask)
110 {
111  void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order,
112  &sg_dma_address(mem), gfp_mask);
113  if (!buf)
114  return -ENOMEM;
115 
116  sg_set_buf(mem, buf, PAGE_SIZE << order);
117  BUG_ON(mem->offset);
118  sg_dma_len(mem) = PAGE_SIZE << order;
119  return 0;
120 }
121 
122 struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
123  gfp_t gfp_mask, int coherent)
124 {
125  struct mlx4_icm *icm;
126  struct mlx4_icm_chunk *chunk = NULL;
127  int cur_order;
128  int ret;
129 
130  /* We use sg_set_buf for coherent allocs, which assumes low memory */
131  BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM));
132 
133  icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
134  if (!icm)
135  return NULL;
136 
137  icm->refcount = 0;
138  INIT_LIST_HEAD(&icm->chunk_list);
139 
140  cur_order = get_order(MLX4_ICM_ALLOC_SIZE);
141 
142  while (npages > 0) {
143  if (!chunk) {
144  chunk = kmalloc(sizeof *chunk,
145  gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
146  if (!chunk)
147  goto fail;
148 
150  chunk->npages = 0;
151  chunk->nsg = 0;
152  list_add_tail(&chunk->list, &icm->chunk_list);
153  }
154 
155  while (1 << cur_order > npages)
156  --cur_order;
157 
158  if (coherent)
159  ret = mlx4_alloc_icm_coherent(&dev->pdev->dev,
160  &chunk->mem[chunk->npages],
161  cur_order, gfp_mask);
162  else
163  ret = mlx4_alloc_icm_pages(&chunk->mem[chunk->npages],
164  cur_order, gfp_mask);
165 
166  if (ret) {
167  if (--cur_order < 0)
168  goto fail;
169  else
170  continue;
171  }
172 
173  ++chunk->npages;
174 
175  if (coherent)
176  ++chunk->nsg;
177  else if (chunk->npages == MLX4_ICM_CHUNK_LEN) {
178  chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
179  chunk->npages,
181 
182  if (chunk->nsg <= 0)
183  goto fail;
184  }
185 
186  if (chunk->npages == MLX4_ICM_CHUNK_LEN)
187  chunk = NULL;
188 
189  npages -= 1 << cur_order;
190  }
191 
192  if (!coherent && chunk) {
193  chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
194  chunk->npages,
196 
197  if (chunk->nsg <= 0)
198  goto fail;
199  }
200 
201  return icm;
202 
203 fail:
204  mlx4_free_icm(dev, icm, coherent);
205  return NULL;
206 }
207 
208 static int mlx4_MAP_ICM(struct mlx4_dev *dev, struct mlx4_icm *icm, u64 virt)
209 {
210  return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM, icm, virt);
211 }
212 
213 static int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count)
214 {
215  return mlx4_cmd(dev, virt, page_count, 0, MLX4_CMD_UNMAP_ICM,
217 }
218 
219 int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm)
220 {
221  return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM_AUX, icm, -1);
222 }
223 
224 int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev)
225 {
226  return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_ICM_AUX,
228 }
229 
230 int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj)
231 {
232  u32 i = (obj & (table->num_obj - 1)) /
233  (MLX4_TABLE_CHUNK_SIZE / table->obj_size);
234  int ret = 0;
235 
236  mutex_lock(&table->mutex);
237 
238  if (table->icm[i]) {
239  ++table->icm[i]->refcount;
240  goto out;
241  }
242 
244  (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
245  __GFP_NOWARN, table->coherent);
246  if (!table->icm[i]) {
247  ret = -ENOMEM;
248  goto out;
249  }
250 
251  if (mlx4_MAP_ICM(dev, table->icm[i], table->virt +
252  (u64) i * MLX4_TABLE_CHUNK_SIZE)) {
253  mlx4_free_icm(dev, table->icm[i], table->coherent);
254  table->icm[i] = NULL;
255  ret = -ENOMEM;
256  goto out;
257  }
258 
259  ++table->icm[i]->refcount;
260 
261 out:
262  mutex_unlock(&table->mutex);
263  return ret;
264 }
265 
266 void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj)
267 {
268  u32 i;
269  u64 offset;
270 
271  i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size);
272 
273  mutex_lock(&table->mutex);
274 
275  if (--table->icm[i]->refcount == 0) {
276  offset = (u64) i * MLX4_TABLE_CHUNK_SIZE;
277  mlx4_UNMAP_ICM(dev, table->virt + offset,
279  mlx4_free_icm(dev, table->icm[i], table->coherent);
280  table->icm[i] = NULL;
281  }
282 
283  mutex_unlock(&table->mutex);
284 }
285 
288 {
289  int offset, dma_offset, i;
290  u64 idx;
291  struct mlx4_icm_chunk *chunk;
292  struct mlx4_icm *icm;
293  struct page *page = NULL;
294 
295  if (!table->lowmem)
296  return NULL;
297 
298  mutex_lock(&table->mutex);
299 
300  idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size;
301  icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE];
302  dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE;
303 
304  if (!icm)
305  goto out;
306 
307  list_for_each_entry(chunk, &icm->chunk_list, list) {
308  for (i = 0; i < chunk->npages; ++i) {
309  if (dma_handle && dma_offset >= 0) {
310  if (sg_dma_len(&chunk->mem[i]) > dma_offset)
311  *dma_handle = sg_dma_address(&chunk->mem[i]) +
312  dma_offset;
313  dma_offset -= sg_dma_len(&chunk->mem[i]);
314  }
315  /*
316  * DMA mapping can merge pages but not split them,
317  * so if we found the page, dma_handle has already
318  * been assigned to.
319  */
320  if (chunk->mem[i].length > offset) {
321  page = sg_page(&chunk->mem[i]);
322  goto out;
323  }
324  offset -= chunk->mem[i].length;
325  }
326  }
327 
328 out:
329  mutex_unlock(&table->mutex);
330  return page ? lowmem_page_address(page) + offset : NULL;
331 }
332 
334  u32 start, u32 end)
335 {
336  int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size;
337  int err;
338  u32 i;
339 
340  for (i = start; i <= end; i += inc) {
341  err = mlx4_table_get(dev, table, i);
342  if (err)
343  goto fail;
344  }
345 
346  return 0;
347 
348 fail:
349  while (i > start) {
350  i -= inc;
351  mlx4_table_put(dev, table, i);
352  }
353 
354  return err;
355 }
356 
358  u32 start, u32 end)
359 {
360  u32 i;
361 
362  for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size)
363  mlx4_table_put(dev, table, i);
364 }
365 
367  u64 virt, int obj_size, u32 nobj, int reserved,
368  int use_lowmem, int use_coherent)
369 {
370  int obj_per_chunk;
371  int num_icm;
372  unsigned chunk_size;
373  int i;
374  u64 size;
375 
376  obj_per_chunk = MLX4_TABLE_CHUNK_SIZE / obj_size;
377  num_icm = (nobj + obj_per_chunk - 1) / obj_per_chunk;
378 
379  table->icm = kcalloc(num_icm, sizeof *table->icm, GFP_KERNEL);
380  if (!table->icm)
381  return -ENOMEM;
382  table->virt = virt;
383  table->num_icm = num_icm;
384  table->num_obj = nobj;
385  table->obj_size = obj_size;
386  table->lowmem = use_lowmem;
387  table->coherent = use_coherent;
388  mutex_init(&table->mutex);
389 
390  size = (u64) nobj * obj_size;
391  for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {
392  chunk_size = MLX4_TABLE_CHUNK_SIZE;
393  if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > size)
394  chunk_size = PAGE_ALIGN(size -
396 
397  table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT,
398  (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
399  __GFP_NOWARN, use_coherent);
400  if (!table->icm[i])
401  goto err;
402  if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) {
403  mlx4_free_icm(dev, table->icm[i], use_coherent);
404  table->icm[i] = NULL;
405  goto err;
406  }
407 
408  /*
409  * Add a reference to this ICM chunk so that it never
410  * gets freed (since it contains reserved firmware objects).
411  */
412  ++table->icm[i]->refcount;
413  }
414 
415  return 0;
416 
417 err:
418  for (i = 0; i < num_icm; ++i)
419  if (table->icm[i]) {
420  mlx4_UNMAP_ICM(dev, virt + i * MLX4_TABLE_CHUNK_SIZE,
421  MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
422  mlx4_free_icm(dev, table->icm[i], use_coherent);
423  }
424 
425  kfree(table->icm);
426 
427  return -ENOMEM;
428 }
429 
431 {
432  int i;
433 
434  for (i = 0; i < table->num_icm; ++i)
435  if (table->icm[i]) {
436  mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,
437  MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
438  mlx4_free_icm(dev, table->icm[i], table->coherent);
439  }
440 
441  kfree(table->icm);
442 }