Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
fmr_pool.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004 Topspin Communications. All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses. You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  * Redistribution and use in source and binary forms, with or
12  * without modification, are permitted provided that the following
13  * conditions are met:
14  *
15  * - Redistributions of source code must retain the above
16  * copyright notice, this list of conditions and the following
17  * disclaimer.
18  *
19  * - Redistributions in binary form must reproduce the above
20  * copyright notice, this list of conditions and the following
21  * disclaimer in the documentation and/or other materials
22  * provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/errno.h>
35 #include <linux/spinlock.h>
36 #include <linux/export.h>
37 #include <linux/slab.h>
38 #include <linux/jhash.h>
39 #include <linux/kthread.h>
40 
41 #include <rdma/ib_fmr_pool.h>
42 
43 #include "core_priv.h"
44 
45 #define PFX "fmr_pool: "
46 
47 enum {
49 
53 };
54 
55 /*
56  * If an FMR is not in use, then the list member will point to either
57  * its pool's free_list (if the FMR can be mapped again; that is,
58  * remap_count < pool->max_remaps) or its pool's dirty_list (if the
59  * FMR needs to be unmapped before being remapped). In either of
60  * these cases it is a bug if the ref_count is not 0. In other words,
61  * if ref_count is > 0, then the list member must not be linked into
62  * either free_list or dirty_list.
63  *
64  * The cache_node member is used to link the FMR into a cache bucket
65  * (if caching is enabled). This is independent of the reference
66  * count of the FMR. When a valid FMR is released, its ref_count is
67  * decremented, and if ref_count reaches 0, the FMR is placed in
68  * either free_list or dirty_list as appropriate. However, it is not
69  * removed from the cache and may be "revived" if a call to
70  * ib_fmr_register_physical() occurs before the FMR is remapped. In
71  * this case we just increment the ref_count and remove the FMR from
72  * free_list/dirty_list.
73  *
74  * Before we remap an FMR from free_list, we remove it from the cache
75  * (to prevent another user from obtaining a stale FMR). When an FMR
76  * is released, we add it to the tail of the free list, so that our
77  * cache eviction policy is "least recently used."
78  *
79  * All manipulation of ref_count, list and cache_node is protected by
80  * pool_lock to maintain consistency.
81  */
82 
83 struct ib_fmr_pool {
85 
86  int pool_size;
87  int max_pages;
90  int dirty_len;
94 
96  void * arg);
97  void *flush_arg;
98 
100 
103 
105 };
106 
107 static inline u32 ib_fmr_hash(u64 first_page)
108 {
109  return jhash_2words((u32) first_page, (u32) (first_page >> 32), 0) &
110  (IB_FMR_HASH_SIZE - 1);
111 }
112 
113 /* Caller must hold pool_lock */
114 static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,
115  u64 *page_list,
116  int page_list_len,
118 {
119  struct hlist_head *bucket;
120  struct ib_pool_fmr *fmr;
121  struct hlist_node *pos;
122 
123  if (!pool->cache_bucket)
124  return NULL;
125 
126  bucket = pool->cache_bucket + ib_fmr_hash(*page_list);
127 
128  hlist_for_each_entry(fmr, pos, bucket, cache_node)
129  if (io_virtual_address == fmr->io_virtual_address &&
130  page_list_len == fmr->page_list_len &&
131  !memcmp(page_list, fmr->page_list,
132  page_list_len * sizeof *page_list))
133  return fmr;
134 
135  return NULL;
136 }
137 
138 static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
139 {
140  int ret;
141  struct ib_pool_fmr *fmr;
142  LIST_HEAD(unmap_list);
143  LIST_HEAD(fmr_list);
144 
145  spin_lock_irq(&pool->pool_lock);
146 
147  list_for_each_entry(fmr, &pool->dirty_list, list) {
148  hlist_del_init(&fmr->cache_node);
149  fmr->remap_count = 0;
150  list_add_tail(&fmr->fmr->list, &fmr_list);
151 
152 #ifdef DEBUG
153  if (fmr->ref_count !=0) {
154  printk(KERN_WARNING PFX "Unmapping FMR 0x%08x with ref count %d\n",
155  fmr, fmr->ref_count);
156  }
157 #endif
158  }
159 
160  list_splice_init(&pool->dirty_list, &unmap_list);
161  pool->dirty_len = 0;
162 
163  spin_unlock_irq(&pool->pool_lock);
164 
165  if (list_empty(&unmap_list)) {
166  return;
167  }
168 
169  ret = ib_unmap_fmr(&fmr_list);
170  if (ret)
171  printk(KERN_WARNING PFX "ib_unmap_fmr returned %d\n", ret);
172 
173  spin_lock_irq(&pool->pool_lock);
174  list_splice(&unmap_list, &pool->free_list);
175  spin_unlock_irq(&pool->pool_lock);
176 }
177 
178 static int ib_fmr_cleanup_thread(void *pool_ptr)
179 {
180  struct ib_fmr_pool *pool = pool_ptr;
181 
182  do {
183  if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {
184  ib_fmr_batch_release(pool);
185 
186  atomic_inc(&pool->flush_ser);
188 
189  if (pool->flush_function)
190  pool->flush_function(pool, pool->flush_arg);
191  }
192 
194  if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&
196  schedule();
198  } while (!kthread_should_stop());
199 
200  return 0;
201 }
202 
212  struct ib_fmr_pool_param *params)
213 {
214  struct ib_device *device;
215  struct ib_fmr_pool *pool;
216  struct ib_device_attr *attr;
217  int i;
218  int ret;
219  int max_remaps;
220 
221  if (!params)
222  return ERR_PTR(-EINVAL);
223 
224  device = pd->device;
225  if (!device->alloc_fmr || !device->dealloc_fmr ||
226  !device->map_phys_fmr || !device->unmap_fmr) {
227  printk(KERN_INFO PFX "Device %s does not support FMRs\n",
228  device->name);
229  return ERR_PTR(-ENOSYS);
230  }
231 
232  attr = kmalloc(sizeof *attr, GFP_KERNEL);
233  if (!attr) {
234  printk(KERN_WARNING PFX "couldn't allocate device attr struct\n");
235  return ERR_PTR(-ENOMEM);
236  }
237 
238  ret = ib_query_device(device, attr);
239  if (ret) {
240  printk(KERN_WARNING PFX "couldn't query device: %d\n", ret);
241  kfree(attr);
242  return ERR_PTR(ret);
243  }
244 
245  if (!attr->max_map_per_fmr)
246  max_remaps = IB_FMR_MAX_REMAPS;
247  else
248  max_remaps = attr->max_map_per_fmr;
249 
250  kfree(attr);
251 
252  pool = kmalloc(sizeof *pool, GFP_KERNEL);
253  if (!pool) {
254  printk(KERN_WARNING PFX "couldn't allocate pool struct\n");
255  return ERR_PTR(-ENOMEM);
256  }
257 
258  pool->cache_bucket = NULL;
259 
260  pool->flush_function = params->flush_function;
261  pool->flush_arg = params->flush_arg;
262 
263  INIT_LIST_HEAD(&pool->free_list);
264  INIT_LIST_HEAD(&pool->dirty_list);
265 
266  if (params->cache) {
267  pool->cache_bucket =
268  kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket,
269  GFP_KERNEL);
270  if (!pool->cache_bucket) {
271  printk(KERN_WARNING PFX "Failed to allocate cache in pool\n");
272  ret = -ENOMEM;
273  goto out_free_pool;
274  }
275 
276  for (i = 0; i < IB_FMR_HASH_SIZE; ++i)
277  INIT_HLIST_HEAD(pool->cache_bucket + i);
278  }
279 
280  pool->pool_size = 0;
281  pool->max_pages = params->max_pages_per_fmr;
282  pool->max_remaps = max_remaps;
283  pool->dirty_watermark = params->dirty_watermark;
284  pool->dirty_len = 0;
285  spin_lock_init(&pool->pool_lock);
286  atomic_set(&pool->req_ser, 0);
287  atomic_set(&pool->flush_ser, 0);
289 
290  pool->thread = kthread_run(ib_fmr_cleanup_thread,
291  pool,
292  "ib_fmr(%s)",
293  device->name);
294  if (IS_ERR(pool->thread)) {
295  printk(KERN_WARNING PFX "couldn't start cleanup thread\n");
296  ret = PTR_ERR(pool->thread);
297  goto out_free_pool;
298  }
299 
300  {
301  struct ib_pool_fmr *fmr;
302  struct ib_fmr_attr fmr_attr = {
303  .max_pages = params->max_pages_per_fmr,
304  .max_maps = pool->max_remaps,
305  .page_shift = params->page_shift
306  };
307  int bytes_per_fmr = sizeof *fmr;
308 
309  if (pool->cache_bucket)
310  bytes_per_fmr += params->max_pages_per_fmr * sizeof (u64);
311 
312  for (i = 0; i < params->pool_size; ++i) {
313  fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);
314  if (!fmr) {
315  printk(KERN_WARNING PFX "failed to allocate fmr "
316  "struct for FMR %d\n", i);
317  goto out_fail;
318  }
319 
320  fmr->pool = pool;
321  fmr->remap_count = 0;
322  fmr->ref_count = 0;
323  INIT_HLIST_NODE(&fmr->cache_node);
324 
325  fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr);
326  if (IS_ERR(fmr->fmr)) {
327  printk(KERN_WARNING PFX "fmr_create failed "
328  "for FMR %d\n", i);
329  kfree(fmr);
330  goto out_fail;
331  }
332 
333  list_add_tail(&fmr->list, &pool->free_list);
334  ++pool->pool_size;
335  }
336  }
337 
338  return pool;
339 
340  out_free_pool:
341  kfree(pool->cache_bucket);
342  kfree(pool);
343 
344  return ERR_PTR(ret);
345 
346  out_fail:
347  ib_destroy_fmr_pool(pool);
348 
349  return ERR_PTR(-ENOMEM);
350 }
352 
360 {
361  struct ib_pool_fmr *fmr;
362  struct ib_pool_fmr *tmp;
363  LIST_HEAD(fmr_list);
364  int i;
365 
366  kthread_stop(pool->thread);
367  ib_fmr_batch_release(pool);
368 
369  i = 0;
370  list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {
371  if (fmr->remap_count) {
372  INIT_LIST_HEAD(&fmr_list);
373  list_add_tail(&fmr->fmr->list, &fmr_list);
374  ib_unmap_fmr(&fmr_list);
375  }
376  ib_dealloc_fmr(fmr->fmr);
377  list_del(&fmr->list);
378  kfree(fmr);
379  ++i;
380  }
381 
382  if (i < pool->pool_size)
383  printk(KERN_WARNING PFX "pool still has %d regions registered\n",
384  pool->pool_size - i);
385 
386  kfree(pool->cache_bucket);
387  kfree(pool);
388 }
390 
398 {
399  int serial;
400  struct ib_pool_fmr *fmr, *next;
401 
402  /*
403  * The free_list holds FMRs that may have been used
404  * but have not been remapped enough times to be dirty.
405  * Put them on the dirty list now so that the cleanup
406  * thread will reap them too.
407  */
408  spin_lock_irq(&pool->pool_lock);
409  list_for_each_entry_safe(fmr, next, &pool->free_list, list) {
410  if (fmr->remap_count > 0)
411  list_move(&fmr->list, &pool->dirty_list);
412  }
413  spin_unlock_irq(&pool->pool_lock);
414 
415  serial = atomic_inc_return(&pool->req_ser);
416  wake_up_process(pool->thread);
417 
419  atomic_read(&pool->flush_ser) - serial >= 0))
420  return -EINTR;
421 
422  return 0;
423 }
425 
435 struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
436  u64 *page_list,
437  int list_len,
438  u64 io_virtual_address)
439 {
440  struct ib_fmr_pool *pool = pool_handle;
441  struct ib_pool_fmr *fmr;
442  unsigned long flags;
443  int result;
444 
445  if (list_len < 1 || list_len > pool->max_pages)
446  return ERR_PTR(-EINVAL);
447 
448  spin_lock_irqsave(&pool->pool_lock, flags);
449  fmr = ib_fmr_cache_lookup(pool,
450  page_list,
451  list_len,
452  io_virtual_address);
453  if (fmr) {
454  /* found in cache */
455  ++fmr->ref_count;
456  if (fmr->ref_count == 1) {
457  list_del(&fmr->list);
458  }
459 
460  spin_unlock_irqrestore(&pool->pool_lock, flags);
461 
462  return fmr;
463  }
464 
465  if (list_empty(&pool->free_list)) {
466  spin_unlock_irqrestore(&pool->pool_lock, flags);
467  return ERR_PTR(-EAGAIN);
468  }
469 
470  fmr = list_entry(pool->free_list.next, struct ib_pool_fmr, list);
471  list_del(&fmr->list);
472  hlist_del_init(&fmr->cache_node);
473  spin_unlock_irqrestore(&pool->pool_lock, flags);
474 
475  result = ib_map_phys_fmr(fmr->fmr, page_list, list_len,
476  io_virtual_address);
477 
478  if (result) {
479  spin_lock_irqsave(&pool->pool_lock, flags);
480  list_add(&fmr->list, &pool->free_list);
481  spin_unlock_irqrestore(&pool->pool_lock, flags);
482 
483  printk(KERN_WARNING PFX "fmr_map returns %d\n", result);
484 
485  return ERR_PTR(result);
486  }
487 
488  ++fmr->remap_count;
489  fmr->ref_count = 1;
490 
491  if (pool->cache_bucket) {
493  fmr->page_list_len = list_len;
494  memcpy(fmr->page_list, page_list, list_len * sizeof(*page_list));
495 
496  spin_lock_irqsave(&pool->pool_lock, flags);
497  hlist_add_head(&fmr->cache_node,
498  pool->cache_bucket + ib_fmr_hash(fmr->page_list[0]));
499  spin_unlock_irqrestore(&pool->pool_lock, flags);
500  }
501 
502  return fmr;
503 }
505 
514 {
515  struct ib_fmr_pool *pool;
516  unsigned long flags;
517 
518  pool = fmr->pool;
519 
520  spin_lock_irqsave(&pool->pool_lock, flags);
521 
522  --fmr->ref_count;
523  if (!fmr->ref_count) {
524  if (fmr->remap_count < pool->max_remaps) {
525  list_add_tail(&fmr->list, &pool->free_list);
526  } else {
527  list_add_tail(&fmr->list, &pool->dirty_list);
528  if (++pool->dirty_len >= pool->dirty_watermark) {
529  atomic_inc(&pool->req_ser);
530  wake_up_process(pool->thread);
531  }
532  }
533  }
534 
535 #ifdef DEBUG
536  if (fmr->ref_count < 0)
537  printk(KERN_WARNING PFX "FMR %p has ref count %d < 0\n",
538  fmr, fmr->ref_count);
539 #endif
540 
541  spin_unlock_irqrestore(&pool->pool_lock, flags);
542 
543  return 0;
544 }