cmpxchg.h

Go to the documentation of this file.

#ifndef _ASM_POWERPC_CMPXCHG_H_
#define _ASM_POWERPC_CMPXCHG_H_

#ifdef __KERNEL__
#include <linux/compiler.h>
#include <asm/synch.h>
#include <asm/asm-compat.h>

/*
 * Atomic exchange
 *
 * Changes the memory location '*ptr' to be val and returns
 * the previous value stored there.
 */
static __always_inline unsigned long
__xchg_u32(volatile void *p, unsigned long val)
{
    unsigned long prev;

    __asm__ __volatile__(
    PPC_RELEASE_BARRIER
"1: lwarx   %0,0,%2 \n"
    PPC405_ERR77(0,%2)
"   stwcx.  %3,0,%2 \n\
    bne-    1b"
    PPC_ACQUIRE_BARRIER
    : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
    : "r" (p), "r" (val)
    : "cc", "memory");

    return prev;
}

/*
 * Atomic exchange
 *
 * Changes the memory location '*ptr' to be val and returns
 * the previous value stored there.
 */
static __always_inline unsigned long
__xchg_u32_local(volatile void *p, unsigned long val)
{
    unsigned long prev;

    __asm__ __volatile__(
"1: lwarx   %0,0,%2 \n"
    PPC405_ERR77(0,%2)
"   stwcx.  %3,0,%2 \n\
    bne-    1b"
    : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
    : "r" (p), "r" (val)
    : "cc", "memory");

    return prev;
}

#ifdef CONFIG_PPC64
static __always_inline unsigned long
__xchg_u64(volatile void *p, unsigned long val)
{
    unsigned long prev;

    __asm__ __volatile__(
    PPC_RELEASE_BARRIER
"1: ldarx   %0,0,%2 \n"
    PPC405_ERR77(0,%2)
"   stdcx.  %3,0,%2 \n\
    bne-    1b"
    PPC_ACQUIRE_BARRIER
    : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
    : "r" (p), "r" (val)
    : "cc", "memory");

    return prev;
}

static __always_inline unsigned long
__xchg_u64_local(volatile void *p, unsigned long val)
{
    unsigned long prev;

    __asm__ __volatile__(
"1: ldarx   %0,0,%2 \n"
    PPC405_ERR77(0,%2)
"   stdcx.  %3,0,%2 \n\
    bne-    1b"
    : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
    : "r" (p), "r" (val)
    : "cc", "memory");

    return prev;
}
#endif

/*
 * This function doesn't exist, so you'll get a linker error
 * if something tries to do an invalid xchg().
 */
extern void __xchg_called_with_bad_pointer(void);

static __always_inline unsigned long
__xchg(volatile void *ptr, unsigned long x, unsigned int size)
{
    switch (size) {
    case 4:
        return __xchg_u32(ptr, x);
#ifdef CONFIG_PPC64
    case 8:
        return __xchg_u64(ptr, x);
#endif
    }
    __xchg_called_with_bad_pointer();
    return x;
}

static __always_inline unsigned long
__xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
{
    switch (size) {
    case 4:
        return __xchg_u32_local(ptr, x);
#ifdef CONFIG_PPC64
    case 8:
        return __xchg_u64_local(ptr, x);
#endif
    }
    __xchg_called_with_bad_pointer();
    return x;
}
#define xchg(ptr,x)                              \
  ({                                         \
     __typeof__(*(ptr)) _x_ = (x);                       \
     (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
  })

#define xchg_local(ptr,x)                            \
  ({                                         \
     __typeof__(*(ptr)) _x_ = (x);                       \
     (__typeof__(*(ptr))) __xchg_local((ptr),                    \
            (unsigned long)_x_, sizeof(*(ptr)));                 \
  })

/*
 * Compare and exchange - if *p == old, set it to new,
 * and return the old value of *p.
 */
#define __HAVE_ARCH_CMPXCHG 1

static __always_inline unsigned long
__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
{
    unsigned int prev;

    __asm__ __volatile__ (
    PPC_RELEASE_BARRIER
"1: lwarx   %0,0,%2     # __cmpxchg_u32\n\
    cmpw    0,%0,%3\n\
    bne-    2f\n"
    PPC405_ERR77(0,%2)
"   stwcx.  %4,0,%2\n\
    bne-    1b"
    PPC_ACQUIRE_BARRIER
    "\n\
2:"
    : "=&r" (prev), "+m" (*p)
    : "r" (p), "r" (old), "r" (new)
    : "cc", "memory");

    return prev;
}

static __always_inline unsigned long
__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
            unsigned long new)
{
    unsigned int prev;

    __asm__ __volatile__ (
"1: lwarx   %0,0,%2     # __cmpxchg_u32\n\
    cmpw    0,%0,%3\n\
    bne-    2f\n"
    PPC405_ERR77(0,%2)
"   stwcx.  %4,0,%2\n\
    bne-    1b"
    "\n\
2:"
    : "=&r" (prev), "+m" (*p)
    : "r" (p), "r" (old), "r" (new)
    : "cc", "memory");

    return prev;
}

#ifdef CONFIG_PPC64
static __always_inline unsigned long
__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
{
    unsigned long prev;

    __asm__ __volatile__ (
    PPC_RELEASE_BARRIER
"1: ldarx   %0,0,%2     # __cmpxchg_u64\n\
    cmpd    0,%0,%3\n\
    bne-    2f\n\
    stdcx.  %4,0,%2\n\
    bne-    1b"
    PPC_ACQUIRE_BARRIER
    "\n\
2:"
    : "=&r" (prev), "+m" (*p)
    : "r" (p), "r" (old), "r" (new)
    : "cc", "memory");

    return prev;
}

static __always_inline unsigned long
__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
            unsigned long new)
{
    unsigned long prev;

    __asm__ __volatile__ (
"1: ldarx   %0,0,%2     # __cmpxchg_u64\n\
    cmpd    0,%0,%3\n\
    bne-    2f\n\
    stdcx.  %4,0,%2\n\
    bne-    1b"
    "\n\
2:"
    : "=&r" (prev), "+m" (*p)
    : "r" (p), "r" (old), "r" (new)
    : "cc", "memory");

    return prev;
}
#endif

/* This function doesn't exist, so you'll get a linker error
   if something tries to do an invalid cmpxchg().  */
extern void __cmpxchg_called_with_bad_pointer(void);

static __always_inline unsigned long
__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
      unsigned int size)
{
    switch (size) {
    case 4:
        return __cmpxchg_u32(ptr, old, new);
#ifdef CONFIG_PPC64
    case 8:
        return __cmpxchg_u64(ptr, old, new);
#endif
    }
    __cmpxchg_called_with_bad_pointer();
    return old;
}

static __always_inline unsigned long
__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
      unsigned int size)
{
    switch (size) {
    case 4:
        return __cmpxchg_u32_local(ptr, old, new);
#ifdef CONFIG_PPC64
    case 8:
        return __cmpxchg_u64_local(ptr, old, new);
#endif
    }
    __cmpxchg_called_with_bad_pointer();
    return old;
}

#define cmpxchg(ptr, o, n)                       \
  ({                                     \
     __typeof__(*(ptr)) _o_ = (o);                   \
     __typeof__(*(ptr)) _n_ = (n);                   \
     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,       \
                    (unsigned long)_n_, sizeof(*(ptr))); \
  })


#define cmpxchg_local(ptr, o, n)                     \
  ({                                     \
     __typeof__(*(ptr)) _o_ = (o);                   \
     __typeof__(*(ptr)) _n_ = (n);                   \
     (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
                    (unsigned long)_n_, sizeof(*(ptr))); \
  })

#ifdef CONFIG_PPC64
#define cmpxchg64(ptr, o, n)                        \
  ({                                    \
    BUILD_BUG_ON(sizeof(*(ptr)) != 8);              \
    cmpxchg((ptr), (o), (n));                   \
  })
#define cmpxchg64_local(ptr, o, n)                  \
  ({                                    \
    BUILD_BUG_ON(sizeof(*(ptr)) != 8);              \
    cmpxchg_local((ptr), (o), (n));                 \
  })
#else
#include <asm-generic/cmpxchg-local.h>
#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
#endif

#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_CMPXCHG_H_ */