radix-tree.h

Go to the documentation of this file.

/*
 * Copyright (C) 2001 Momchil Velikov
 * Portions Copyright (C) 2001 Christoph Hellwig
 * Copyright (C) 2006 Nick Piggin
 * Copyright (C) 2012 Konstantin Khlebnikov
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2, or (at
 * your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
#ifndef _LINUX_RADIX_TREE_H
#define _LINUX_RADIX_TREE_H

#include <linux/preempt.h>
#include <linux/types.h>
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/rcupdate.h>

/*
 * An indirect pointer (root->rnode pointing to a radix_tree_node, rather
 * than a data item) is signalled by the low bit set in the root->rnode
 * pointer.
 *
 * In this case root->height is > 0, but the indirect pointer tests are
 * needed for RCU lookups (because root->height is unreliable). The only
 * time callers need worry about this is when doing a lookup_slot under
 * RCU.
 *
 * Indirect pointer in fact is also used to tag the last pointer of a node
 * when it is shrunk, before we rcu free the node. See shrink code for
 * details.
 */
#define RADIX_TREE_INDIRECT_PTR     1
/*
 * A common use of the radix tree is to store pointers to struct pages;
 * but shmem/tmpfs needs also to store swap entries in the same tree:
 * those are marked as exceptional entries to distinguish them.
 * EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it.
 */
#define RADIX_TREE_EXCEPTIONAL_ENTRY    2
#define RADIX_TREE_EXCEPTIONAL_SHIFT    2

static inline int radix_tree_is_indirect_ptr(void *ptr)
{
    return (int)((unsigned long)ptr & RADIX_TREE_INDIRECT_PTR);
}

/*** radix-tree API starts here ***/

#define RADIX_TREE_MAX_TAGS 3

/* root tags are stored in gfp_mask, shifted by __GFP_BITS_SHIFT */
struct radix_tree_root {
    unsigned int        height;
    gfp_t           gfp_mask;
    struct radix_tree_node  __rcu *rnode;
};

#define RADIX_TREE_INIT(mask)   {                   \
    .height = 0,                            \
    .gfp_mask = (mask),                     \
    .rnode = NULL,                          \
}

#define RADIX_TREE(name, mask) \
    struct radix_tree_root name = RADIX_TREE_INIT(mask)

#define INIT_RADIX_TREE(root, mask)                 \
do {                                    \
    (root)->height = 0;                     \
    (root)->gfp_mask = (mask);                  \
    (root)->rnode = NULL;                       \
} while (0)

static inline void *radix_tree_deref_slot(void **pslot)
{
    return rcu_dereference(*pslot);
}

static inline void *radix_tree_deref_slot_protected(void **pslot,
                            spinlock_t *treelock)
{
    return rcu_dereference_protected(*pslot, lockdep_is_held(treelock));
}

static inline int radix_tree_deref_retry(void *arg)
{
    return unlikely((unsigned long)arg & RADIX_TREE_INDIRECT_PTR);
}

static inline int radix_tree_exceptional_entry(void *arg)
{
    /* Not unlikely because radix_tree_exception often tested first */
    return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY;
}

static inline int radix_tree_exception(void *arg)
{
    return unlikely((unsigned long)arg &
        (RADIX_TREE_INDIRECT_PTR | RADIX_TREE_EXCEPTIONAL_ENTRY));
}

static inline void radix_tree_replace_slot(void **pslot, void *item)
{
    BUG_ON(radix_tree_is_indirect_ptr(item));
    rcu_assign_pointer(*pslot, item);
}

int radix_tree_insert(struct radix_tree_root *, unsigned long, void *);
void *radix_tree_lookup(struct radix_tree_root *, unsigned long);
void **radix_tree_lookup_slot(struct radix_tree_root *, unsigned long);
void *radix_tree_delete(struct radix_tree_root *, unsigned long);
unsigned int
radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
            unsigned long first_index, unsigned int max_items);
unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
            void ***results, unsigned long *indices,
            unsigned long first_index, unsigned int max_items);
unsigned long radix_tree_next_hole(struct radix_tree_root *root,
                unsigned long index, unsigned long max_scan);
unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
                unsigned long index, unsigned long max_scan);
int radix_tree_preload(gfp_t gfp_mask);
void radix_tree_init(void);
void *radix_tree_tag_set(struct radix_tree_root *root,
            unsigned long index, unsigned int tag);
void *radix_tree_tag_clear(struct radix_tree_root *root,
            unsigned long index, unsigned int tag);
int radix_tree_tag_get(struct radix_tree_root *root,
            unsigned long index, unsigned int tag);
unsigned int
radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
        unsigned long first_index, unsigned int max_items,
        unsigned int tag);
unsigned int
radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
        unsigned long first_index, unsigned int max_items,
        unsigned int tag);
unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
        unsigned long *first_indexp, unsigned long last_index,
        unsigned long nr_to_tag,
        unsigned int fromtag, unsigned int totag);
int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);
unsigned long radix_tree_locate_item(struct radix_tree_root *root, void *item);

static inline void radix_tree_preload_end(void)
{
    preempt_enable();
}

struct radix_tree_iter {
    unsigned long   index;
    unsigned long   next_index;
    unsigned long   tags;
};

#define RADIX_TREE_ITER_TAG_MASK    0x00FF  /* tag index in lower byte */
#define RADIX_TREE_ITER_TAGGED      0x0100  /* lookup tagged slots */
#define RADIX_TREE_ITER_CONTIG      0x0200  /* stop at first hole */

static __always_inline void **
radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start)
{
    /*
     * Leave iter->tags uninitialized. radix_tree_next_chunk() will fill it
     * in the case of a successful tagged chunk lookup.  If the lookup was
     * unsuccessful or non-tagged then nobody cares about ->tags.
     *
     * Set index to zero to bypass next_index overflow protection.
     * See the comment in radix_tree_next_chunk() for details.
     */
    iter->index = 0;
    iter->next_index = start;
    return NULL;
}

void **radix_tree_next_chunk(struct radix_tree_root *root,
                 struct radix_tree_iter *iter, unsigned flags);

static __always_inline unsigned
radix_tree_chunk_size(struct radix_tree_iter *iter)
{
    return iter->next_index - iter->index;
}

static __always_inline void **
radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, unsigned flags)
{
    if (flags & RADIX_TREE_ITER_TAGGED) {
        iter->tags >>= 1;
        if (likely(iter->tags & 1ul)) {
            iter->index++;
            return slot + 1;
        }
        if (!(flags & RADIX_TREE_ITER_CONTIG) && likely(iter->tags)) {
            unsigned offset = __ffs(iter->tags);

            iter->tags >>= offset;
            iter->index += offset + 1;
            return slot + offset + 1;
        }
    } else {
        unsigned size = radix_tree_chunk_size(iter) - 1;

        while (size--) {
            slot++;
            iter->index++;
            if (likely(*slot))
                return slot;
            if (flags & RADIX_TREE_ITER_CONTIG) {
                /* forbid switching to the next chunk */
                iter->next_index = 0;
                break;
            }
        }
    }
    return NULL;
}

#define radix_tree_for_each_chunk(slot, root, iter, start, flags)   \
    for (slot = radix_tree_iter_init(iter, start) ;         \
          (slot = radix_tree_next_chunk(root, iter, flags)) ;)

#define radix_tree_for_each_chunk_slot(slot, iter, flags)       \
    for (; slot ; slot = radix_tree_next_slot(slot, iter, flags))

#define radix_tree_for_each_slot(slot, root, iter, start)       \
    for (slot = radix_tree_iter_init(iter, start) ;         \
         slot || (slot = radix_tree_next_chunk(root, iter, 0)) ;    \
         slot = radix_tree_next_slot(slot, iter, 0))

#define radix_tree_for_each_contig(slot, root, iter, start)     \
    for (slot = radix_tree_iter_init(iter, start) ;         \
         slot || (slot = radix_tree_next_chunk(root, iter,      \
                RADIX_TREE_ITER_CONTIG)) ;      \
         slot = radix_tree_next_slot(slot, iter,            \
                RADIX_TREE_ITER_CONTIG))

#define radix_tree_for_each_tagged(slot, root, iter, start, tag)    \
    for (slot = radix_tree_iter_init(iter, start) ;         \
         slot || (slot = radix_tree_next_chunk(root, iter,      \
                  RADIX_TREE_ITER_TAGGED | tag)) ;      \
         slot = radix_tree_next_slot(slot, iter,            \
                RADIX_TREE_ITER_TAGGED))

#endif /* _LINUX_RADIX_TREE_H */