@@ -9,62 +9,71 @@
* tables.
*/
-#ifndef __IDR_H__
-#define __IDR_H__
+#ifndef _LINUX_IDR_H
+#define _LINUX_IDR_H
#include <linux/radix-tree.h>
#include <linux/gfp.h>
#include <linux/percpu.h>
+#include <linux/xarray.h>
struct idr {
- struct radix_tree_root idr_rt;
+ struct xarray idxa;
};
-/*
- * The IDR API does not expose the tagging functionality of the radix tree
- * to users. Use tag 0 to track whether a node has free space below it.
- */
-#define IDR_FREE 0
-
-/* Set the IDR flag and the IDR_FREE tag */
-#define IDR_RT_MARKER ((__force gfp_t)(3 << __GFP_BITS_SHIFT))
+#define IDR_INIT_FLAGS XA_FLAGS_TRACK_FREE | XA_FLAGS_TAG(0)
#define IDR_INIT(name) \
{ \
- .idr_rt = RADIX_TREE_INIT(name, IDR_RT_MARKER) \
+ .idxa = __XARRAY_INIT(name.idxa, IDR_INIT_FLAGS) \
}
#define DEFINE_IDR(name) struct idr name = IDR_INIT(name)
+static inline void idr_init(struct idr *idr)
+{
+ __xa_init(&idr->idxa, IDR_INIT_FLAGS);
+}
+
/**
* DOC: idr sync
- * idr synchronization (stolen from radix-tree.h)
+ * idr synchronization
*
- * idr_find() is able to be called locklessly, using RCU. The caller must
- * ensure calls to this function are made within rcu_read_lock() regions.
- * Other readers (lock-free or otherwise) and modifications may be running
- * concurrently.
+ * The IDR manages its own locking, using irqsafe spinlocks for operations
+ * which modify the IDR and RCU for operations which do not. The user of
+ * the IDR may choose to wrap accesses to it in a lock if it needs to
+ * guarantee the IDR does not change during a read access. The easiest way
+ * to do this is to grab the same lock the IDR uses for write accesses
+ * using one of the idr_lock() wrappers.
*
- * It is still required that the caller manage the synchronization and
- * lifetimes of the items. So if RCU lock-free lookups are used, typically
- * this would mean that the items have their own locks, or are amenable to
- * lock-free access; and that the items are freed by RCU (or only freed after
- * having been deleted from the idr tree *and* a synchronize_rcu() grace
- * period).
+ * The caller must still manage the synchronization and lifetimes of the
+ * items. So if RCU lock-free lookups are used, typically this would mean
+ * that the items have their own locks, or are amenable to lock-free access;
+ * and that the items are freed by RCU (or only freed after having been
+ * deleted from the IDR *and* a synchronize_rcu() grace period has elapsed).
*/
-void idr_preload(gfp_t gfp_mask);
+#define idr_lock(idr) xa_lock(&(idr)->idxa)
+#define idr_unlock(idr) xa_unlock(&(idr)->idxa)
+#define idr_lock_bh(idr) xa_lock_bh(&(idr)->idxa)
+#define idr_unlock_bh(idr) xa_unlock_bh(&(idr)->idxa)
+#define idr_lock_irq(idr) xa_lock_irq(&(idr)->idxa)
+#define idr_unlock_irq(idr) xa_unlock_irq(&(idr)->idxa)
+#define idr_lock_irqsave(idr, flags) \
+ xa_lock_irqsave(&(idr)->idxa, flags)
+#define idr_unlock_irqrestore(idr, flags) \
+ xa_unlock_irqrestore(&(idr)->idxa, flags)
+
+void idr_preload(gfp_t);
int idr_alloc(struct idr *, void *, int start, int end, gfp_t);
int __must_check idr_alloc_ul(struct idr *, void *, unsigned long *nextid,
unsigned long max, gfp_t);
int idr_alloc_cyclic(struct idr *, int *cursor, void *entry,
int start, int end, gfp_t);
-int idr_for_each(const struct idr *,
+int idr_for_each(struct idr *,
int (*fn)(int id, void *p, void *data), void *data);
void *idr_get_next(struct idr *, int *nextid);
-void *idr_get_next_ul(struct idr *, unsigned long *nextid);
void *idr_replace(struct idr *, void *, unsigned long id);
-void idr_destroy(struct idr *);
static inline int __must_check idr_alloc_u32(struct idr *idr, void *ptr,
u32 *nextid, unsigned long max, gfp_t gfp)
@@ -77,22 +86,21 @@ static inline int __must_check idr_alloc_u32(struct idr *idr, void *ptr,
static inline void *idr_remove(struct idr *idr, unsigned long id)
{
- return radix_tree_delete_item(&idr->idr_rt, id, NULL);
+ return xa_store(&idr->idxa, id, NULL, GFP_NOWAIT);
}
-static inline void idr_init(struct idr *idr)
+static inline bool idr_is_empty(const struct idr *idr)
{
- INIT_RADIX_TREE(&idr->idr_rt, IDR_RT_MARKER);
+ return xa_empty(&idr->idxa);
}
-static inline bool idr_is_empty(const struct idr *idr)
+static inline void idr_destroy(struct idr *idr)
{
- return radix_tree_empty(&idr->idr_rt) &&
- radix_tree_tagged(&idr->idr_rt, IDR_FREE);
+ xa_destroy(&idr->idxa);
}
/**
- * idr_preload_end - end preload section started with idr_preload()
+ * idr_preload_end() - end preload section started with idr_preload()
*
* Each idr_preload() should be matched with an invocation of this
* function. See idr_preload() for details.
@@ -103,7 +111,7 @@ static inline void idr_preload_end(void)
}
/**
- * idr_find - return pointer for given id
+ * idr_find() - return pointer for given id
* @idr: idr handle
* @id: lookup key
*
@@ -111,12 +119,11 @@ static inline void idr_preload_end(void)
* return indicates that @id is not valid or you passed %NULL in
* idr_get_new().
*
- * This function can be called under rcu_read_lock(), given that the leaf
- * pointers lifetimes are correctly managed.
+ * This function is protected by the RCU read lock.
*/
-static inline void *idr_find(const struct idr *idr, unsigned long id)
+static inline void *idr_find(struct idr *idr, unsigned long id)
{
- return radix_tree_lookup(&idr->idr_rt, id);
+ return xa_load(&idr->idxa, id);
}
/**
@@ -132,7 +139,7 @@ static inline void *idr_find(const struct idr *idr, unsigned long id)
#define idr_for_each_entry(idr, entry, id) \
for (id = 0; ((entry) = idr_get_next(idr, &(id))) != NULL; ++id)
#define idr_for_each_entry_ul(idr, entry, id) \
- for (id = 0; ((entry) = idr_get_next_ul(idr, &(id))) != NULL; ++id)
+ xa_for_each(&(idr)->idxa, entry, id, ULONG_MAX)
/**
* idr_for_each_entry_continue - continue iteration over an idr's elements of a given type
@@ -167,7 +174,7 @@ struct ida {
};
#define IDA_INIT(name) { \
- .ida_rt = RADIX_TREE_INIT(name, IDR_RT_MARKER | GFP_NOWAIT), \
+ .ida_rt = RADIX_TREE_INIT(name, IDR_INIT_FLAGS | GFP_NOWAIT), \
}
#define DEFINE_IDA(name) struct ida name = IDA_INIT(name)
@@ -182,7 +189,7 @@ void ida_simple_remove(struct ida *ida, unsigned int id);
static inline void ida_init(struct ida *ida)
{
- INIT_RADIX_TREE(&ida->ida_rt, IDR_RT_MARKER | GFP_NOWAIT);
+ INIT_RADIX_TREE(&ida->ida_rt, IDR_INIT_FLAGS | GFP_NOWAIT);
}
/**
@@ -201,4 +208,4 @@ static inline bool ida_is_empty(const struct ida *ida)
{
return radix_tree_empty(&ida->ida_rt);
}
-#endif /* __IDR_H__ */
+#endif /* _LINUX_IDR_H */
@@ -362,7 +362,13 @@ static inline bool xa_is_sibling(void *entry)
(entry < xa_mk_sibling(XA_CHUNK_SIZE - 1));
}
-#define XA_RETRY_ENTRY xa_mk_internal(256)
+#define XA_SKIP_ENTRY xa_mk_internal(256)
+#define XA_RETRY_ENTRY xa_mk_internal(257)
+
+static inline bool xa_is_skip(void *entry)
+{
+ return unlikely(entry == XA_SKIP_ENTRY);
+}
static inline bool xa_is_retry(void *entry)
{
@@ -444,18 +450,20 @@ static inline bool xas_not_node(struct xa_node *node)
}
/**
- * xas_retry() - Handle a retry entry.
+ * xas_retry() - Retry the operation if appropriate.
* @xas: XArray operation state.
* @entry: Entry from xarray.
*
- * An RCU-protected read may see a retry entry as a side-effect of a
- * simultaneous modification. This function sets up the @xas to retry
- * the walk from the head of the array.
+ * The advanced functions may sometimes return an internal entry, such as
+ * a retry entry or a skip entry. This function sets up the @xas to restart
+ * the walk from the head of the array if needed.
*
* Return: true if the operation needs to be retried.
*/
static inline bool xas_retry(struct xa_state *xas, void *entry)
{
+ if (xa_is_skip(entry))
+ return true;
if (!xa_is_retry(entry))
return false;
xas->xa_node = XAS_RESTART;
@@ -8,6 +8,9 @@
DEFINE_PER_CPU(struct ida_bitmap *, ida_bitmap);
static DEFINE_SPINLOCK(simple_ida_lock);
+/* In radix-tree.c temporarily */
+extern bool idr_nomem(struct xa_state *, gfp_t);
+
/**
* idr_alloc_ul() - allocate a large ID
* @idr: idr handle
@@ -29,30 +32,36 @@ static DEFINE_SPINLOCK(simple_ida_lock);
int idr_alloc_ul(struct idr *idr, void *ptr, unsigned long *nextid,
unsigned long max, gfp_t gfp)
{
- struct radix_tree_iter iter;
- void __rcu **slot;
+ XA_STATE(xas, *nextid);
+ unsigned long flags;
+ int err;
- if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))
+ if (WARN_ON_ONCE(xa_is_internal(ptr)))
return -EINVAL;
-
- if (WARN_ON_ONCE(!(idr->idr_rt.xa_flags & ROOT_IS_IDR)))
- idr->idr_rt.xa_flags |= IDR_RT_MARKER;
-
- radix_tree_iter_init(&iter, *nextid);
- slot = idr_get_free(&idr->idr_rt, &iter, gfp, max);
- if (IS_ERR(slot))
- return PTR_ERR(slot);
-
- radix_tree_iter_replace(&idr->idr_rt, &iter, slot, ptr);
- radix_tree_iter_tag_clear(&idr->idr_rt, &iter, IDR_FREE);
-
- *nextid = iter.index;
- return 0;
+ if (!ptr)
+ ptr = XA_SKIP_ENTRY;
+
+ do {
+ xa_lock_irqsave(&idr->idxa, flags);
+ xas_find_tag(&idr->idxa, &xas, max, XA_FREE_TAG);
+ if (xas.xa_index > max)
+ xas_set_err(&xas, ENOSPC);
+ xas_store(&idr->idxa, &xas, ptr);
+ xas_clear_tag(&idr->idxa, &xas, XA_FREE_TAG);
+ xa_unlock_irqrestore(&idr->idxa, flags);
+ } while (idr_nomem(&xas, gfp));
+
+ err = xas_error(&xas);
+ if (!err)
+ *nextid = xas.xa_index;
+ xas_destroy(&xas);
+
+ return err;
}
EXPORT_SYMBOL_GPL(idr_alloc_ul);
/**
- * idr_alloc - allocate an id
+ * idr_alloc() - allocate an id
* @idr: idr handle
* @ptr: pointer to be associated with the new id
* @start: the minimum id (inclusive)
@@ -117,10 +126,10 @@ int idr_alloc_cyclic(struct idr *idr, int *cursor, void *ptr,
return id;
}
-EXPORT_SYMBOL(idr_alloc_cyclic);
+EXPORT_SYMBOL_GPL(idr_alloc_cyclic);
/**
- * idr_for_each - iterate through all stored pointers
+ * idr_for_each() - iterate through all stored pointers
* @idr: idr handle
* @fn: function to be called for each pointer
* @data: data passed to callback function
@@ -136,69 +145,41 @@ EXPORT_SYMBOL(idr_alloc_cyclic);
* seen and deleted entries may be seen, but adding and removing entries
* will not cause other entries to be skipped, nor spurious ones to be seen.
*/
-int idr_for_each(const struct idr *idr,
+int idr_for_each(struct idr *idr,
int (*fn)(int id, void *p, void *data), void *data)
{
- struct radix_tree_iter iter;
- void __rcu **slot;
+ unsigned long i = 0;
+ void *p;
- radix_tree_for_each_slot(slot, &idr->idr_rt, &iter, 0) {
- int ret = fn(iter.index, rcu_dereference_raw(*slot), data);
+ xa_for_each(&idr->idxa, p, i, INT_MAX) {
+ int ret = fn(i, p, data);
if (ret)
return ret;
}
return 0;
}
-EXPORT_SYMBOL(idr_for_each);
-
-/**
- * idr_get_next - Find next populated entry
- * @idr: idr handle
- * @nextid: Pointer to lowest possible ID to return
- *
- * Returns the next populated entry in the tree with an ID greater than
- * or equal to the value pointed to by @nextid. On exit, @nextid is updated
- * to the ID of the found value. To use in a loop, the value pointed to by
- * nextid must be incremented by the user.
- */
-void *idr_get_next(struct idr *idr, int *nextid)
-{
- struct radix_tree_iter iter;
- void __rcu **slot;
-
- slot = radix_tree_iter_find(&idr->idr_rt, &iter, *nextid);
- if (!slot)
- return NULL;
-
- *nextid = iter.index;
- return rcu_dereference_raw(*slot);
-}
-EXPORT_SYMBOL(idr_get_next);
+EXPORT_SYMBOL_GPL(idr_for_each);
/**
- * idr_get_next_ul - Find next populated entry
+ * idr_get_next() - Find next populated entry
* @idr: idr handle
- * @nextid: Pointer to lowest possible ID to return
+ * @id: Pointer to lowest possible ID to return
*
* Returns the next populated entry in the tree with an ID greater than
* or equal to the value pointed to by @nextid. On exit, @nextid is updated
* to the ID of the found value. To use in a loop, the value pointed to by
* nextid must be incremented by the user.
*/
-void *idr_get_next_ul(struct idr *idr, unsigned long *nextid)
+void *idr_get_next(struct idr *idr, int *id)
{
- struct radix_tree_iter iter;
- void __rcu **slot;
+ unsigned long index = *id;
+ void *entry = xa_find(&idr->idxa, &index, INT_MAX);
- slot = radix_tree_iter_find(&idr->idr_rt, &iter, *nextid);
- if (!slot)
- return NULL;
-
- *nextid = iter.index;
- return rcu_dereference_raw(*slot);
+ *id = index;
+ return entry;
}
-EXPORT_SYMBOL(idr_get_next_ul);
+EXPORT_SYMBOL_GPL(idr_get_next);
/**
* idr_replace - replace pointer for given id
@@ -216,22 +197,28 @@ EXPORT_SYMBOL(idr_get_next_ul);
*/
void *idr_replace(struct idr *idr, void *ptr, unsigned long id)
{
- struct radix_tree_node *node;
- void __rcu **slot = NULL;
- void *entry;
+ XA_STATE(xas, id);
+ unsigned long flags;
+ void *curr;
- if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))
+ if (WARN_ON_ONCE(xa_is_internal(ptr)))
return ERR_PTR(-EINVAL);
-
- entry = __radix_tree_lookup(&idr->idr_rt, id, &node, &slot);
- if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE))
- return ERR_PTR(-ENOENT);
-
- __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL);
-
- return entry;
+ if (!ptr)
+ ptr = XA_SKIP_ENTRY;
+
+ xa_lock_irqsave(&idr->idxa, flags);
+ curr = xas_load(&idr->idxa, &xas);
+ if (curr)
+ xas_store(&idr->idxa, &xas, ptr);
+ else
+ curr = ERR_PTR(-ENOENT);
+ xa_unlock_irqrestore(&idr->idxa, flags);
+
+ if (xa_is_skip(curr))
+ return NULL;
+ return curr;
}
-EXPORT_SYMBOL(idr_replace);
+EXPORT_SYMBOL_GPL(idr_replace);
/**
* DOC: IDA description
@@ -262,7 +249,7 @@ EXPORT_SYMBOL(idr_replace);
* Developer's notes:
*
* The IDA uses the functionality provided by the IDR & radix tree to store
- * bitmaps in each entry. The IDR_FREE tag means there is at least one bit
+ * bitmaps in each entry. The XA_FREE_TAG tag means there is at least one bit
* free, unlike the IDR where it means at least one entry is free.
*
* I considered telling the radix tree that each slot is an order-10 node
@@ -368,7 +355,7 @@ int ida_get_new_above(struct ida *ida, int start, int *id)
__set_bit(bit, bitmap->bitmap);
if (bitmap_full(bitmap->bitmap, IDA_BITMAP_BITS))
radix_tree_iter_tag_clear(root, &iter,
- IDR_FREE);
+ XA_FREE_TAG);
} else {
new += bit;
if (new < 0)
@@ -424,7 +411,7 @@ void ida_remove(struct ida *ida, int id)
goto err;
__clear_bit(offset, btmp);
- radix_tree_iter_tag_set(&ida->ida_rt, &iter, IDR_FREE);
+ radix_tree_iter_tag_set(&ida->ida_rt, &iter, XA_FREE_TAG);
if (xa_is_value(bitmap)) {
if (xa_to_value(rcu_dereference_raw(*slot)) == 0)
radix_tree_iter_delete(&ida->ida_rt, &iter, slot);
@@ -572,6 +572,28 @@ int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order)
return __radix_tree_preload(gfp_mask, nr_nodes);
}
+/* Once the IDR users abandon the preload API, we can use xas_nomem */
+bool idr_nomem(struct xa_state *xas, gfp_t gfp)
+{
+ if (xas->xa_node != XAS_ERROR(ENOMEM))
+ return false;
+ xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep,
+ gfp | __GFP_NOWARN);
+ if (!xas->xa_alloc) {
+ struct radix_tree_preload *rtp;
+
+ rtp = this_cpu_ptr(&radix_tree_preloads);
+ if (!rtp->nr)
+ return false;
+ xas->xa_alloc = rtp->nodes;
+ rtp->nodes = xas->xa_alloc->parent;
+ rtp->nr--;
+ }
+
+ xas->xa_node = XAS_RESTART;
+ return true;
+}
+
static unsigned radix_tree_load_root(const struct radix_tree_root *root,
struct radix_tree_node **nodep, unsigned long *maxindex)
{
@@ -605,7 +627,7 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp,
maxshift += RADIX_TREE_MAP_SHIFT;
entry = rcu_dereference_raw(root->xa_head);
- if (!entry && (!is_idr(root) || root_tag_get(root, IDR_FREE)))
+ if (!entry && (!is_idr(root) || root_tag_get(root, XA_FREE_TAG)))
goto out;
do {
@@ -615,10 +637,10 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp,
return -ENOMEM;
if (is_idr(root)) {
- all_tag_set(node, IDR_FREE);
- if (!root_tag_get(root, IDR_FREE)) {
- rtag_clear(node, IDR_FREE, 0);
- root_tag_set(root, IDR_FREE);
+ all_tag_set(node, XA_FREE_TAG);
+ if (!root_tag_get(root, XA_FREE_TAG)) {
+ rtag_clear(node, XA_FREE_TAG, 0);
+ root_tag_set(root, XA_FREE_TAG);
}
} else {
/* Propagate the aggregated tag info to the new child */
@@ -689,8 +711,8 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root,
* one (root->xa_head) as far as dependent read barriers go.
*/
root->xa_head = (void __rcu *)child;
- if (is_idr(root) && !rtag_get(node, IDR_FREE, 0))
- root_tag_clear(root, IDR_FREE);
+ if (is_idr(root) && !rtag_get(node, XA_FREE_TAG, 0))
+ root_tag_clear(root, XA_FREE_TAG);
/*
* We have a dilemma here. The node's slot[0] must not be
@@ -1117,7 +1139,7 @@ static bool node_tag_get(const struct radix_tree_root *root,
/*
* IDR users want to be able to store NULL in the tree, so if the slot isn't
* free, don't adjust the count, even if it's transitioning between NULL and
- * non-NULL. For the IDA, we mark slots as being IDR_FREE while they still
+ * non-NULL. For the IDA, we mark slots as being XA_FREE_TAG while they still
* have empty bits, but it only stores NULL in slots when they're being
* deleted.
*/
@@ -1127,7 +1149,7 @@ static int calculate_count(struct radix_tree_root *root,
{
if (is_idr(root)) {
unsigned offset = get_slot_offset(node, slot);
- bool free = node_tag_get(root, node, IDR_FREE, offset);
+ bool free = node_tag_get(root, node, XA_FREE_TAG, offset);
if (!free)
return 0;
if (!old)
@@ -1958,7 +1980,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root,
int tag;
if (is_idr(root))
- node_tag_set(root, node, IDR_FREE, offset);
+ node_tag_set(root, node, XA_FREE_TAG, offset);
else
for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
node_tag_clear(root, node, tag, offset);
@@ -2006,7 +2028,7 @@ void *radix_tree_delete_item(struct radix_tree_root *root,
void *entry;
entry = __radix_tree_lookup(root, index, &node, &slot);
- if (!entry && (!is_idr(root) || node_tag_get(root, node, IDR_FREE,
+ if (!entry && (!is_idr(root) || node_tag_get(root, node, XA_FREE_TAG,
get_slot_offset(node, slot))))
return NULL;
@@ -2113,7 +2135,7 @@ void __rcu **idr_get_free(struct radix_tree_root *root,
grow:
shift = radix_tree_load_root(root, &child, &maxindex);
- if (!radix_tree_tagged(root, IDR_FREE))
+ if (!radix_tree_tagged(root, XA_FREE_TAG))
start = max(start, maxindex + 1);
if (start > max)
return ERR_PTR(-ENOSPC);
@@ -2134,7 +2156,7 @@ void __rcu **idr_get_free(struct radix_tree_root *root,
offset, 0, 0);
if (!child)
return ERR_PTR(-ENOMEM);
- all_tag_set(child, IDR_FREE);
+ all_tag_set(child, XA_FREE_TAG);
rcu_assign_pointer(*slot, node_to_entry(child));
if (node)
node->count++;
@@ -2143,8 +2165,8 @@ void __rcu **idr_get_free(struct radix_tree_root *root,
node = entry_to_node(child);
offset = radix_tree_descend(node, &child, start);
- if (!rtag_get(node, IDR_FREE, offset)) {
- offset = radix_tree_find_next_bit(node, IDR_FREE,
+ if (!rtag_get(node, XA_FREE_TAG, offset)) {
+ offset = radix_tree_find_next_bit(node, XA_FREE_TAG,
offset + 1);
start = rnext_index(start, node, offset);
if (start > max)
@@ -2168,32 +2190,11 @@ void __rcu **idr_get_free(struct radix_tree_root *root,
iter->next_index = 1;
iter->node = node;
__set_iter_shift(iter, shift);
- set_iter_tags(iter, node, offset, IDR_FREE);
+ set_iter_tags(iter, node, offset, XA_FREE_TAG);
return slot;
}
-/**
- * idr_destroy - release all internal memory from an IDR
- * @idr: idr handle
- *
- * After this function is called, the IDR is empty, and may be reused or
- * the data structure containing it may be freed.
- *
- * A typical clean-up sequence for objects stored in an idr tree will use
- * idr_for_each() to free all objects, if necessary, then idr_destroy() to
- * free the memory used to keep track of those objects.
- */
-void idr_destroy(struct idr *idr)
-{
- struct radix_tree_node *node = rcu_dereference_raw(idr->idr_rt.xa_head);
- if (radix_tree_is_internal_node(node))
- radix_tree_free_nodes(node);
- idr->idr_rt.xa_head = NULL;
- root_tag_set(&idr->idr_rt, IDR_FREE);
-}
-EXPORT_SYMBOL(idr_destroy);
-
static void
radix_tree_node_ctor(void *arg)
{
@@ -1078,6 +1078,8 @@ void *xa_load(struct xarray *xa, unsigned long index)
rcu_read_lock();
do {
entry = xas_load(xa, &xas);
+ if (xa_is_skip(entry))
+ entry = NULL;
} while (xas_retry(&xas, entry));
rcu_read_unlock();
@@ -1113,6 +1115,8 @@ void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
xa_lock_irqsave(xa, flags);
curr = xas_store(xa, &xas, entry);
xa_unlock_irqrestore(xa, flags);
+ if (xa_is_skip(curr))
+ curr = NULL;
} while (xas_nomem(&xas, gfp));
xas_destroy(&xas);