@@ -1,11 +1,80 @@
#ifndef _LINUX_MMAP_LOCK_H
#define _LINUX_MMAP_LOCK_H
+#include <linux/histogram.h>
+#include <linux/mm_types.h>
#include <linux/mmdebug.h>
+#include <linux/types.h>
+#include <linux/sched/clock.h>
#define MMAP_LOCK_INITIALIZER(name) \
.mmap_lock = __RWSEM_INITIALIZER(name.mmap_lock),
+#ifdef CONFIG_MMAP_LOCK_HISTOGRAMS
+static inline void __mmap_lock_histogram_record_duration(struct mm_struct *mm,
+ u64 duration)
+{
+ if (likely(mm->mmap_lock_contention))
+ histogram_record_rcu(mm->mmap_lock_contention, duration, 1);
+}
+
+static inline void mmap_lock_histogram_record(struct mm_struct *mm,
+ u64 start_time_ns)
+{
+ __mmap_lock_histogram_record_duration(mm,
+ sched_clock() - start_time_ns);
+}
+#endif
+
+static inline bool __mmap_trylock(struct mm_struct *mm,
+ int (*trylock)(struct rw_semaphore *))
+{
+ bool ret = trylock(&mm->mmap_lock) != 0;
+
+#ifdef CONFIG_MMAP_LOCK_HISTOGRAMS
+ if (ret)
+ __mmap_lock_histogram_record_duration(mm, 0);
+#endif
+ return ret;
+}
+
+static inline void __mmap_lock(struct mm_struct *mm,
+ int (*trylock)(struct rw_semaphore *),
+ void (*lock)(struct rw_semaphore *))
+{
+#ifdef CONFIG_MMAP_LOCK_HISTOGRAMS
+ u64 start_time_ns;
+
+ if (!__mmap_trylock(mm, trylock)) {
+ start_time_ns = sched_clock();
+ lock(&mm->mmap_lock);
+ mmap_lock_histogram_record(mm, start_time_ns);
+ }
+#else
+ lock(&mm->mmap_lock);
+#endif
+}
+
+static inline int __mmap_lock_return(struct mm_struct *mm,
+ int (*trylock)(struct rw_semaphore *),
+ int (*lock)(struct rw_semaphore *))
+{
+#ifdef CONFIG_MMAP_LOCK_HISTOGRAMS
+ u64 start_time_ns;
+ int ret;
+
+ if (!__mmap_trylock(mm, trylock)) {
+ start_time_ns = sched_clock();
+ ret = lock(&mm->mmap_lock);
+ mmap_lock_histogram_record(mm, start_time_ns);
+ return ret;
+ }
+ return 0;
+#else
+ return lock(&mm->mmap_lock);
+#endif
+}
+
static inline void mmap_init_lock(struct mm_struct *mm)
{
init_rwsem(&mm->mmap_lock);
@@ -13,22 +82,26 @@ static inline void mmap_init_lock(struct mm_struct *mm)
static inline void mmap_write_lock(struct mm_struct *mm)
{
- down_write(&mm->mmap_lock);
+ __mmap_lock(mm, down_write_trylock, down_write);
}
+#ifdef CONFIG_MMAP_LOCK_HISTOGRAMS
+void mmap_write_lock_nested(struct mm_struct *mm, int subclass);
+#else
static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
{
down_write_nested(&mm->mmap_lock, subclass);
}
+#endif
static inline int mmap_write_lock_killable(struct mm_struct *mm)
{
- return down_write_killable(&mm->mmap_lock);
+ return __mmap_lock_return(mm, down_write_trylock, down_write_killable);
}
static inline bool mmap_write_trylock(struct mm_struct *mm)
{
- return down_write_trylock(&mm->mmap_lock) != 0;
+ return __mmap_trylock(mm, down_write_trylock);
}
static inline void mmap_write_unlock(struct mm_struct *mm)
@@ -43,17 +116,17 @@ static inline void mmap_write_downgrade(struct mm_struct *mm)
static inline void mmap_read_lock(struct mm_struct *mm)
{
- down_read(&mm->mmap_lock);
+ __mmap_lock(mm, down_read_trylock, down_read);
}
static inline int mmap_read_lock_killable(struct mm_struct *mm)
{
- return down_read_killable(&mm->mmap_lock);
+ return __mmap_lock_return(mm, down_read_trylock, down_read_killable);
}
static inline bool mmap_read_trylock(struct mm_struct *mm)
{
- return down_read_trylock(&mm->mmap_lock) != 0;
+ return __mmap_trylock(mm, down_read_trylock);
}
static inline void mmap_read_unlock(struct mm_struct *mm)
@@ -63,7 +136,7 @@ static inline void mmap_read_unlock(struct mm_struct *mm)
static inline bool mmap_read_trylock_non_owner(struct mm_struct *mm)
{
- if (down_read_trylock(&mm->mmap_lock)) {
+ if (__mmap_trylock(mm, down_read_trylock)) {
rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
return true;
}
@@ -1380,7 +1380,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
/*
* lock for writing
*/
-static inline void __down_write(struct rw_semaphore *sem)
+void __down_write(struct rw_semaphore *sem)
{
long tmp = RWSEM_UNLOCKED_VALUE;
@@ -1405,7 +1405,7 @@ static inline int __down_write_killable(struct rw_semaphore *sem)
return 0;
}
-static inline int __down_write_trylock(struct rw_semaphore *sem)
+int __down_write_trylock(struct rw_semaphore *sem)
{
long tmp;
@@ -112,3 +112,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
+obj-$(CONFIG_MMAP_LOCK_HISTOGRAMS) += mmap_lock.o
new file mode 100644
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/kernel.h>
+#include <linux/lockdep.h>
+#include <linux/mmap_lock.h>
+
+#define MMAP_LOCK_CONTENDED(_mm, try, lock) \
+ do { \
+ if (!try(&(_mm)->mmap_lock)) { \
+ u64 start_time_ns; \
+ lock_contended(&(_mm)->mmap_lock.dep_map, _RET_IP_); \
+ start_time_ns = sched_clock(); \
+ lock(&(_mm)->mmap_lock); \
+ mmap_lock_histogram_record(_mm, start_time_ns); \
+ } else { \
+ __mmap_lock_histogram_record_duration(_mm, 0); \
+ } \
+ lock_acquired(&(_mm)->mmap_lock.dep_map, _RET_IP_); \
+ } while (0)
+
+/* Defined in kernel/locking/rwsem.c. */
+extern int __down_write_trylock(struct rw_semaphore *sem);
+extern void __down_write(struct rw_semaphore *sem);
+
+void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
+{
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+ might_sleep();
+ rwsem_acquire(&mm->mmap_lock.dep_map, subclass, 0, _RET_IP_);
+ MMAP_LOCK_CONTENDED(mm, __down_write_trylock, __down_write);
+#else
+ mmap_write_lock(mm);
+#endif
+}
+EXPORT_SYMBOL(mmap_write_lock_nested);
To minimize overhead, we call down_{read,write}_trylock first. On success, record the acquisition with "0" latency. On failure, we call e.g. down_{read,write} and record the actual measured latency. The reason for recording uncontended acquisitions is so we have an accurate count of total acquisition attempts. This allows us to compute meaningful percentiles over the histogram data - e.g., "x% of lock acquisitions succeeded in <= y ms". Note that in e.g. the killable lock case, we record the latency even if the acquisition doesn't actually succeed. This is so we can see cases where we waited a long time, even if acquisition eventually ended up failing. Nested locks are a weird case, in that we can't do the "only compute latency in the contended case" optimization without reaching into rwsem.c's internal API. The approach this commit uses (removing "static inline" from the couple of functions we need, and forward declaring them in mmap_lock.c) was chosen instead of: - Adding mmap_lock specific code to rwsem.c directly, as this would pollute a generic file with a specific use case's implementation. - Moving the API internals we need up into rwsem.h, as this would expose them more widely than arguably we'd like. - Just computing latency in all cases, as this would add nontrivial overhead to the uncontended lock case. Signed-off-by: Axel Rasmussen <axelrasmussen@google.com> --- include/linux/mmap_lock.h | 87 +++++++++++++++++++++++++++++++++++---- kernel/locking/rwsem.c | 4 +- mm/Makefile | 1 + mm/mmap_lock.c | 34 +++++++++++++++ 4 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 mm/mmap_lock.c