27 #include <linux/list.h>
34 #define POOL_SENTINEL 0x87658765
35 #define OBJ_SENTINEL 0x12345678
36 #define OBJNODE_SENTINEL 0xfedcba09
44 static void tmem_objnode_tree_init(
void);
48 tmem_objnode_tree_init();
88 switch (tmem_oid_compare(oidp, &obj->
oid)) {
92 rbnode = &(*rbnode)->rb_left;
95 rbnode = &(*rbnode)->rb_right;
115 return __tmem_obj_find(hb, oidp,
NULL,
NULL);
118 static void tmem_pampd_destroy_all_in_obj(
struct tmem_obj *);
131 tmem_pampd_destroy_all_in_obj(obj);
138 tmem_oid_set_invalid(&obj->
oid);
163 if (__tmem_obj_find(hb, oidp, &parent, &
new))
188 spin_lock(&hb->
lock);
190 while (rbnode !=
NULL) {
193 tmem_pampd_destroy_all_in_obj(obj);
194 tmem_obj_free(obj, hb);
197 spin_unlock(&hb->
lock);
225 static void tmem_objnode_tree_init(
void)
227 unsigned int ht,
tmp;
229 for (ht = 0; ht <
ARRAY_SIZE(tmem_objnode_tree_h2max); ht++) {
232 tmem_objnode_tree_h2max[ht] = ~0
UL;
234 tmem_objnode_tree_h2max[ht] =
258 static void tmem_objnode_free(
struct tmem_objnode *objnode)
270 pool = objnode->
obj->pool;
273 objnode->
obj->objnode_count--;
283 unsigned int height, shift;
310 return slot !=
NULL ? (
void **)slot :
NULL;
317 slot = (
struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index);
321 static void *tmem_pampd_replace_in_obj(
struct tmem_obj *obj,
uint32_t index,
327 slot = (
struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index);
328 if ((slot !=
NULL) && (*slot !=
NULL)) {
329 void *old_pampd = *(
void **)slot;
330 *(
void **)slot = new_pampd;
342 unsigned int height, shift;
348 if (index > tmem_objnode_tree_h2max[height])
349 while (index > tmem_objnode_tree_h2max[height])
356 newnode = tmem_objnode_alloc(obj);
362 newnode->slots_in_use = 1;
374 slot = tmem_objnode_alloc(obj);
404 static void *tmem_pampd_delete_from_obj(
struct tmem_obj *obj,
uint32_t index)
409 unsigned int height, shift;
417 if (index > tmem_objnode_tree_h2max[height])
436 }
while (height > 0);
441 pathp->
objnode->slots_in_use--;
442 if (pathp->
objnode->slots_in_use) {
455 tmem_objnode_free(to_free);
460 tmem_objnode_free(pathp->
objnode);
474 static void tmem_objnode_node_destroy(
struct tmem_obj *obj,
483 if (objnode->
slots[i]) {
491 tmem_objnode_node_destroy(obj, objnode->
slots[i], ht-1);
492 tmem_objnode_free(objnode->
slots[i]);
498 static void tmem_pampd_destroy_all_in_obj(
struct tmem_obj *obj)
538 void *pampd =
NULL, *pampd_del =
NULL;
543 spin_lock(&hb->
lock);
544 obj = objfound = tmem_obj_find(hb, oidp);
546 pampd = tmem_pampd_lookup_in_obj(objfound, index);
549 pampd_del = tmem_pampd_delete_from_obj(obj, index);
550 BUG_ON(pampd_del != pampd);
564 tmem_obj_init(obj, hb, pool, oidp);
567 BUG_ON(((objnew != obj) && (objfound != obj)) || (objnew == objfound));
572 ret = tmem_pampd_add_to_obj(obj, index, pampd);
575 goto delete_and_free;
579 (
void)tmem_pampd_delete_from_obj(obj, index);
584 tmem_obj_free(objnew, hb);
588 spin_unlock(&hb->
lock);
605 char *
data,
size_t *
size,
bool raw,
int get_and_free)
612 bool free = (get_and_free == 1) || ((get_and_free == 0) && ephemeral);
613 bool lock_held =
false;
616 spin_lock(&hb->
lock);
618 obj = tmem_obj_find(hb, oidp);
622 pampd = tmem_pampd_delete_from_obj(obj, index);
624 pampd = tmem_pampd_lookup_in_obj(obj, index);
629 tmem_obj_free(obj, hb);
636 spin_unlock(&hb->
lock);
640 data, size, raw, pampd, pool, oidp, index);
643 data, size, raw, pampd, pool, oidp, index);
649 spin_unlock(&hb->
lock);
667 spin_lock(&hb->
lock);
668 obj = tmem_obj_find(hb, oidp);
671 pampd = tmem_pampd_delete_from_obj(obj, index);
676 tmem_obj_free(obj, hb);
682 spin_unlock(&hb->
lock);
699 spin_lock(&hb->
lock);
700 obj = tmem_obj_find(hb, oidp);
703 new_pampd = tmem_pampd_replace_in_obj(obj, index, new_pampd);
706 spin_unlock(&hb->
lock);
720 spin_lock(&hb->
lock);
721 obj = tmem_obj_find(hb, oidp);
724 tmem_pampd_destroy_all_in_obj(obj);
725 tmem_obj_free(obj, hb);
730 spin_unlock(&hb->
lock);
744 tmem_pool_flush(pool, 1);