35 #include <linux/list.h>
45 #define POOL_SENTINEL 0x87658765
46 #define OBJ_SENTINEL 0x12345678
47 #define OBJNODE_SENTINEL 0xfedcba09
55 static void tmem_objnode_tree_init(
void);
59 tmem_objnode_tree_init();
106 switch (tmem_oid_compare(oidp, &obj->
oid)) {
110 rbnode = &(*rbnode)->rb_left;
113 rbnode = &(*rbnode)->rb_right;
130 return __tmem_obj_find(hb, oidp,
NULL,
NULL);
133 static void tmem_pampd_destroy_all_in_obj(
struct tmem_obj *,
bool);
146 tmem_pampd_destroy_all_in_obj(obj,
false);
153 tmem_oid_set_invalid(&obj->
oid);
175 #ifdef CONFIG_RAMSTER
181 if (__tmem_obj_find(hb, oidp, &parent, &
new))
206 spin_lock(&hb->
lock);
208 while (rbnode !=
NULL) {
211 tmem_pampd_destroy_all_in_obj(obj,
true);
212 tmem_obj_free(obj, hb);
215 spin_unlock(&hb->
lock);
243 static void tmem_objnode_tree_init(
void)
245 unsigned int ht,
tmp;
247 for (ht = 0; ht <
ARRAY_SIZE(tmem_objnode_tree_h2max); ht++) {
250 tmem_objnode_tree_h2max[ht] = ~0
UL;
252 tmem_objnode_tree_h2max[ht] =
276 static void tmem_objnode_free(
struct tmem_objnode *objnode)
288 pool = objnode->
obj->pool;
291 objnode->
obj->objnode_count--;
301 unsigned int height, shift;
328 return slot !=
NULL ? (
void **)slot :
NULL;
335 slot = (
struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index);
339 #ifdef CONFIG_RAMSTER
340 static void *tmem_pampd_replace_in_obj(
struct tmem_obj *obj,
uint32_t index,
341 void *new_pampd,
bool no_free)
346 slot = (
struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index);
347 if ((slot !=
NULL) && (*slot !=
NULL)) {
348 void *old_pampd = *(
void **)slot;
349 *(
void **)slot = new_pampd;
364 unsigned int height, shift;
370 if (index > tmem_objnode_tree_h2max[height])
371 while (index > tmem_objnode_tree_h2max[height])
378 newnode = tmem_objnode_alloc(obj);
384 newnode->slots_in_use = 1;
396 slot = tmem_objnode_alloc(obj);
426 static void *tmem_pampd_delete_from_obj(
struct tmem_obj *obj,
uint32_t index)
431 unsigned int height, shift;
439 if (index > tmem_objnode_tree_h2max[height])
458 }
while (height > 0);
463 pathp->
objnode->slots_in_use--;
464 if (pathp->
objnode->slots_in_use) {
477 tmem_objnode_free(to_free);
482 tmem_objnode_free(pathp->
objnode);
496 static void tmem_objnode_node_destroy(
struct tmem_obj *obj,
505 if (objnode->
slots[i]) {
513 tmem_objnode_node_destroy(obj, objnode->
slots[i], ht-1);
514 tmem_objnode_free(objnode->
slots[i]);
520 static void tmem_pampd_destroy_all_in_obj(
struct tmem_obj *obj,
536 #ifdef CONFIG_RAMSTER
561 bool raw,
void *pampd_to_use)
564 void *pampd =
NULL, *pampd_del =
NULL;
569 spin_lock(&hb->
lock);
570 obj = objfound = tmem_obj_find(hb, oidp);
572 pampd = tmem_pampd_lookup_in_obj(objfound, index);
575 pampd_del = tmem_pampd_delete_from_obj(obj, index);
576 BUG_ON(pampd_del != pampd);
590 tmem_obj_init(obj, hb, pool, oidp);
593 BUG_ON(((objnew != obj) && (objfound != obj)) || (objnew == objfound));
594 pampd = pampd_to_use;
596 ret = tmem_pampd_add_to_obj(obj, index, pampd);
599 goto delete_and_free;
604 (
void)tmem_pampd_delete_from_obj(obj, index);
608 tmem_obj_free(objnew, hb);
612 spin_unlock(&hb->
lock);
616 #ifdef CONFIG_RAMSTER
635 spin_lock(&hb->
lock);
636 obj = tmem_obj_find(hb, oidp);
638 pampd = tmem_pampd_lookup_in_obj(obj, index);
640 *saved_hb = (
void *)hb;
646 void *pampd,
void *saved_hb,
bool delete)
653 (
void)tmem_pampd_replace_in_obj(obj, index, pampd, 1);
657 (
void)tmem_pampd_delete_from_obj(obj, index);
659 spin_unlock(&hb->
lock);
669 void *old_pampd = *ppampd, *new_pampd =
NULL;
670 bool intransit =
false;
675 old_pampd, pool, oidp, index, &intransit);
678 else if (new_pampd !=
NULL)
681 spin_unlock(&hb->
lock);
683 ret = (*
tmem_pamops.repatriate)(old_pampd, new_pampd, pool,
684 oidp, index, free, data);
690 }
else if (ret != 0 && ret != -
ENOENT) {
710 spin_lock(&hb->
lock);
711 obj = tmem_obj_find(hb, oidp);
714 new_pampd = tmem_pampd_replace_in_obj(obj, index, new_pampd, 0);
719 spin_unlock(&hb->
lock);
736 char *data,
size_t *sizep,
bool raw,
int get_and_free)
743 bool free = (get_and_free == 1) || ((get_and_free == 0) && ephemeral);
744 bool lock_held =
false;
749 spin_lock(&hb->
lock);
751 obj = tmem_obj_find(hb, oidp);
754 ppampd = __tmem_pampd_lookup_in_obj(obj, index);
757 #ifdef CONFIG_RAMSTER
760 ret = tmem_repatriate(ppampd, hb, pool, oidp,
771 pampd = tmem_pampd_delete_from_obj(obj, index);
773 pampd = tmem_pampd_lookup_in_obj(obj, index);
778 tmem_obj_free(obj, hb);
785 data, sizep, raw, pampd, pool, oidp, index);
788 data, sizep, raw, pampd, pool, oidp, index);
794 spin_unlock(&hb->
lock);
812 spin_lock(&hb->
lock);
813 obj = tmem_obj_find(hb, oidp);
816 pampd = tmem_pampd_delete_from_obj(obj, index);
821 tmem_obj_free(obj, hb);
827 spin_unlock(&hb->
lock);
841 spin_lock(&hb->
lock);
842 obj = tmem_obj_find(hb, oidp);
845 tmem_pampd_destroy_all_in_obj(obj,
false);
846 tmem_obj_free(obj, hb);
851 spin_unlock(&hb->
lock);
865 tmem_pool_flush(pool, 1);