barrier.h

Go to the documentation of this file.

#ifndef _ASM_X86_BARRIER_H
#define _ASM_X86_BARRIER_H

#include <asm/alternative.h>
#include <asm/nops.h>

/*
 * Force strict CPU ordering.
 * And yes, this is required on UP too when we're talking
 * to devices.
 */

#ifdef CONFIG_X86_32
/*
 * Some non-Intel clones support out of order store. wmb() ceases to be a
 * nop for these.
 */
#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
#else
#define mb()    asm volatile("mfence":::"memory")
#define rmb()   asm volatile("lfence":::"memory")
#define wmb()   asm volatile("sfence" ::: "memory")
#endif

#define read_barrier_depends()  do { } while (0)

#ifdef CONFIG_SMP
#define smp_mb()    mb()
#ifdef CONFIG_X86_PPRO_FENCE
# define smp_rmb()  rmb()
#else
# define smp_rmb()  barrier()
#endif
#ifdef CONFIG_X86_OOSTORE
# define smp_wmb()  wmb()
#else
# define smp_wmb()  barrier()
#endif
#define smp_read_barrier_depends()  read_barrier_depends()
#define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
#else
#define smp_mb()    barrier()
#define smp_rmb()   barrier()
#define smp_wmb()   barrier()
#define smp_read_barrier_depends()  do { } while (0)
#define set_mb(var, value) do { var = value; barrier(); } while (0)
#endif

/*
 * Stop RDTSC speculation. This is needed when you need to use RDTSC
 * (or get_cycles or vread that possibly accesses the TSC) in a defined
 * code region.
 *
 * (Could use an alternative three way for this if there was one.)
 */
static __always_inline void rdtsc_barrier(void)
{
    alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
    alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
}

#endif /* _ASM_X86_BARRIER_H */