xarray: Add XArray conditional store operations

Like cmpxchg(), xa_cmpxchg will only store to the index if the current
entry matches the old entry.  It returns the current entry, which is
usually more useful than the errno returned by radix_tree_insert().
For the users who really only want the errno, the xa_insert() wrapper
provides a more convenient calling convention.

Signed-off-by: Matthew Wilcox <willy@infradead.org>

diff --git a/include/linux/xarray.h b/include/linux/xarray.h
index 15eab63286ed184f314c1c577276233d38c6b701..7f740f675b96c3605c7a96dee7619461c87a73fd 100644 (file)
--- a/include/linux/xarray.h
+++ b/include/linux/xarray.h
@@ -275,6 +275,8 @@ struct xarray {
 void xa_init_flags(struct xarray *, gfp_t flags);
 void *xa_load(struct xarray *, unsigned long index);
 void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
+void *xa_cmpxchg(struct xarray *, unsigned long index,
+                       void *old, void *entry, gfp_t);
 bool xa_get_mark(struct xarray *, unsigned long index, xa_mark_t);
 void xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
 void xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
@@ -334,6 +336,34 @@ static inline void *xa_erase(struct xarray *xa, unsigned long index)
        return xa_store(xa, index, NULL, 0);
 }
 
+/**
+ * xa_insert() - Store this entry in the XArray unless another entry is
+ *                     already present.
+ * @xa: XArray.
+ * @index: Index into array.
+ * @entry: New entry.
+ * @gfp: Memory allocation flags.
+ *
+ * If you would rather see the existing entry in the array, use xa_cmpxchg().
+ * This function is for users who don't care what the entry is, only that
+ * one is present.
+ *
+ * Context: Process context.  Takes and releases the xa_lock.
+ *         May sleep if the @gfp flags permit.
+ * Return: 0 if the store succeeded.  -EEXIST if another entry was present.
+ * -ENOMEM if memory could not be allocated.
+ */
+static inline int xa_insert(struct xarray *xa, unsigned long index,
+               void *entry, gfp_t gfp)
+{
+       void *curr = xa_cmpxchg(xa, index, NULL, entry, gfp);
+       if (!curr)
+               return 0;
+       if (xa_is_err(curr))
+               return xa_err(curr);
+       return -EEXIST;
+}
+
 #define xa_trylock(xa)         spin_trylock(&(xa)->xa_lock)
 #define xa_lock(xa)            spin_lock(&(xa)->xa_lock)
 #define xa_unlock(xa)          spin_unlock(&(xa)->xa_lock)
@@ -355,9 +385,39 @@ static inline void *xa_erase(struct xarray *xa, unsigned long index)
  */
 void *__xa_erase(struct xarray *, unsigned long index);
 void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
+void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old,
+               void *entry, gfp_t);
 void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
 void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
 
+/**
+ * __xa_insert() - Store this entry in the XArray unless another entry is
+ *                     already present.
+ * @xa: XArray.
+ * @index: Index into array.
+ * @entry: New entry.
+ * @gfp: Memory allocation flags.
+ *
+ * If you would rather see the existing entry in the array, use __xa_cmpxchg().
+ * This function is for users who don't care what the entry is, only that
+ * one is present.
+ *
+ * Context: Any context.  Expects xa_lock to be held on entry.  May
+ *         release and reacquire xa_lock if the @gfp flags permit.
+ * Return: 0 if the store succeeded.  -EEXIST if another entry was present.
+ * -ENOMEM if memory could not be allocated.
+ */
+static inline int __xa_insert(struct xarray *xa, unsigned long index,
+               void *entry, gfp_t gfp)
+{
+       void *curr = __xa_cmpxchg(xa, index, NULL, entry, gfp);
+       if (!curr)
+               return 0;
+       if (xa_is_err(curr))
+               return xa_err(curr);
+       return -EEXIST;
+}
+
 /**
  * xa_erase_bh() - Erase this entry from the XArray.
  * @xa: XArray.

diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index b711030fbc32d420c56aa308a23a99822b186594..fb472258b639dbb822157e3e525e2a61d51a7088 100644 (file)
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -198,6 +198,25 @@ static noinline void check_xa_shrink(struct xarray *xa)
        XA_BUG_ON(xa, !xa_empty(xa));
 }
 
+static noinline void check_cmpxchg(struct xarray *xa)
+{
+       void *FIVE = xa_mk_value(5);
+       void *SIX = xa_mk_value(6);
+       void *LOTS = xa_mk_value(12345678);
+
+       XA_BUG_ON(xa, !xa_empty(xa));
+       XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL);
+       XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST);
+       XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS);
+       XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS);
+       XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE);
+       XA_BUG_ON(xa, xa_cmpxchg(xa, 5, FIVE, NULL, GFP_KERNEL) != NULL);
+       XA_BUG_ON(xa, xa_cmpxchg(xa, 5, NULL, FIVE, GFP_KERNEL) != NULL);
+       xa_erase_index(xa, 12345678);
+       xa_erase_index(xa, 5);
+       XA_BUG_ON(xa, !xa_empty(xa));
+}
+
 static noinline void check_multi_store(struct xarray *xa)
 {
 #ifdef CONFIG_XARRAY_MULTI
@@ -274,6 +293,7 @@ static int xarray_checks(void)
        check_xa_load(&array);
        check_xa_mark(&array);
        check_xa_shrink(&array);
+       check_cmpxchg(&array);
        check_multi_store(&array);
 
        printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);

diff --git a/lib/xarray.c b/lib/xarray.c
index 4596a95ed9cda761dc0db79a9ef629ce5f5fcd21..2ba5a98ec6186e4d22da97d9c86a2b03e3f18bc1 100644 (file)
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -975,6 +975,77 @@ void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
 }
 EXPORT_SYMBOL(__xa_store);
 
+/**
+ * xa_cmpxchg() - Conditionally replace an entry in the XArray.
+ * @xa: XArray.
+ * @index: Index into array.
+ * @old: Old value to test against.
+ * @entry: New value to place in array.
+ * @gfp: Memory allocation flags.
+ *
+ * If the entry at @index is the same as @old, replace it with @entry.
+ * If the return value is equal to @old, then the exchange was successful.
+ *
+ * Context: Process context.  Takes and releases the xa_lock.  May sleep
+ * if the @gfp flags permit.
+ * Return: The old value at this index or xa_err() if an error happened.
+ */
+void *xa_cmpxchg(struct xarray *xa, unsigned long index,
+                       void *old, void *entry, gfp_t gfp)
+{
+       XA_STATE(xas, xa, index);
+       void *curr;
+
+       if (WARN_ON_ONCE(xa_is_internal(entry)))
+               return XA_ERROR(-EINVAL);
+
+       do {
+               xas_lock(&xas);
+               curr = xas_load(&xas);
+               if (curr == old)
+                       xas_store(&xas, entry);
+               xas_unlock(&xas);
+       } while (xas_nomem(&xas, gfp));
+
+       return xas_result(&xas, curr);
+}
+EXPORT_SYMBOL(xa_cmpxchg);
+
+/**
+ * __xa_cmpxchg() - Store this entry in the XArray.
+ * @xa: XArray.
+ * @index: Index into array.
+ * @old: Old value to test against.
+ * @entry: New entry.
+ * @gfp: Memory allocation flags.
+ *
+ * You must already be holding the xa_lock when calling this function.
+ * It will drop the lock if needed to allocate memory, and then reacquire
+ * it afterwards.
+ *
+ * Context: Any context.  Expects xa_lock to be held on entry.  May
+ * release and reacquire xa_lock if @gfp flags permit.
+ * Return: The old entry at this index or xa_err() if an error happened.
+ */
+void *__xa_cmpxchg(struct xarray *xa, unsigned long index,
+                       void *old, void *entry, gfp_t gfp)
+{
+       XA_STATE(xas, xa, index);
+       void *curr;
+
+       if (WARN_ON_ONCE(xa_is_internal(entry)))
+               return XA_ERROR(-EINVAL);
+
+       do {
+               curr = xas_load(&xas);
+               if (curr == old)
+                       xas_store(&xas, entry);
+       } while (__xas_nomem(&xas, gfp));
+
+       return xas_result(&xas, curr);
+}
+EXPORT_SYMBOL(__xa_cmpxchg);
+
 /**
  * __xa_set_mark() - Set this mark on this entry while locked.
  * @xa: XArray.