@@ -60,19 +60,27 @@
static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
{
- short inc = 0x0100;
+ short inc;
asm volatile (
+ "1:\t\n"
+ "mov $0x100, %0\n\t"
LOCK_PREFIX "xaddw %w0, %1\n"
- "1:\t"
+ "2:\t"
"cmpb %h0, %b0\n\t"
- "je 2f\n\t"
+ "je 4f\n\t"
+ "3:\t\n"
"rep ; nop\n\t"
- "movb %1, %b0\n\t"
/* don't need lfence here, because loads are in-order */
- "jmp 1b\n"
- "2:"
- : "+Q" (inc), "+m" (lock->slock)
+ ALTERNATIVE(
+ "movb %1, %b0\n\t"
+ "jmp 2b\n",
+ "nop", X86_FEATURE_HYPERVISOR)"\n\t"
+ "cmpw $0, %1\n\t"
+ "jne 3b\n\t"
+ "jmp 1b\n\t"
+ "4:"
+ : "=Q" (inc), "+m" (lock->slock)
:
: "memory", "cc");
}
@@ -98,10 +106,13 @@ static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
{
- asm volatile(UNLOCK_LOCK_PREFIX "incb %0"
- : "+m" (lock->slock)
- :
- : "memory", "cc");
+ asm volatile(
+ ALTERNATIVE(UNLOCK_LOCK_PREFIX "incb (%0);"ASM_NOP3,
+ UNLOCK_LOCK_PREFIX "movw $0, (%0)",
+ X86_FEATURE_HYPERVISOR)
+ :
+ : "Q" (&lock->slock)
+ : "memory", "cc");
}
#else
#define TICKET_SHIFT 16