@@ -66,20 +66,18 @@ static inline int __tickets_equal(__ticket_t one, __ticket_t two)
return !((one ^ two) & ~TICKET_SLOWPATH_FLAG);
}
-static inline void __ticket_check_and_clear_slowpath(arch_spinlock_t *lock,
- __ticket_t head)
+static inline void __ticket_clear_slowpath(arch_spinlock_t *lock,
+ __ticket_t head)
{
- if (head & TICKET_SLOWPATH_FLAG) {
- arch_spinlock_t old, new;
+ arch_spinlock_t old, new;
- old.tickets.head = head;
- new.tickets.head = head & ~TICKET_SLOWPATH_FLAG;
- old.tickets.tail = new.tickets.head + TICKET_LOCK_INC;
- new.tickets.tail = old.tickets.tail;
+ old.tickets.head = head;
+ new.tickets.head = head & ~TICKET_SLOWPATH_FLAG;
+ old.tickets.tail = new.tickets.head + TICKET_LOCK_INC;
+ new.tickets.tail = old.tickets.tail;
- /* try to clear slowpath flag when there are no contenders */
- cmpxchg(&lock->head_tail, old.head_tail, new.head_tail);
- }
+ /* try to clear slowpath flag when there are no contenders */
+ cmpxchg(&lock->head_tail, old.head_tail, new.head_tail);
}
static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
@@ -120,7 +118,8 @@ static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
__ticket_lock_spinning(lock, inc.tail);
}
clear_slowpath:
- __ticket_check_and_clear_slowpath(lock, inc.head);
+ if (unlikely(inc.head & TICKET_SLOWPATH_FLAG))
+ __ticket_clear_slowpath(lock, inc.head);
out:
barrier(); /* make sure nothing creeps before the lock is taken */
}
The decision whether the slowpath flag is to be cleared for paravirtualized spinlocks is located in __ticket_check_and_clear_slowpath() today. Move that decision into arch_spin_lock() and add an unlikely attribute to it to avoid calling a function in case the compiler chooses not to inline __ticket_check_and_clear_slowpath() and the slowpath flag isn't set. Signed-off-by: Juergen Gross <jgross@suse.com> --- arch/x86/include/asm/spinlock.h | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-)