@@ -41,6 +41,15 @@ static DEFINE_PER_CPU(u32, seed);
*/
#define INTRA_NODE_HANDOFF_PROB_ARG 0x10000
+/*
+ * Controls the probability for enabling the scan of the main queue when
+ * the secondary queue is empty. The chosen value reduces the amount of
+ * unnecessary shuffling of threads between the two waiting queues when
+ * the contention is low, while responding fast enough and enabling
+ * the shuffling when the contention is high.
+ */
+#define SHUFFLE_REDUCTION_PROB_ARG 0x80
+
static inline __pure int decode_numa_node(u32 node_and_count)
{
int node = (node_and_count >> _Q_NODE_OFFSET) - 1;
@@ -197,6 +206,18 @@ static inline void pass_mcs_lock(struct mcs_spinlock *node,
struct mcs_spinlock *succ = NULL;
/*
+ * Limit thread shuffling when the secondary queue is empty.
+ * This copes with the overhead the shuffling creates when the
+ * lock is only lightly contended, and threads do not stay
+ * in the secondary queue long enough to reap the benefit of moving
+ * them there.
+ */
+ if (node->locked == 1 && probably(SHUFFLE_REDUCTION_PROB_ARG)) {
+ arch_mcs_spin_unlock_contended(&next->locked, 1);
+ return;
+ }
+
+ /*
* Try to pass the lock to a thread running on the same node.
* For long-term fairness, search for such a thread with high
* probability rather than always.