@@ -233,6 +233,7 @@ struct mmu_notifier {
struct mm_struct *mm;
struct rcu_head rcu;
unsigned int users;
+ bool has_fast_aging;
};
/**
@@ -387,6 +388,7 @@ extern int __mmu_notifier_clear_young(struct mm_struct *mm,
extern int __mmu_notifier_test_young(struct mm_struct *mm,
unsigned long address,
bool fast_only);
+extern bool __mm_has_fast_young_notifiers(struct mm_struct *mm);
extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);
extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r);
extern void __mmu_notifier_arch_invalidate_secondary_tlbs(struct mm_struct *mm,
@@ -449,6 +451,13 @@ static inline int mmu_notifier_test_young_fast_only(struct mm_struct *mm,
return 0;
}
+static inline bool mm_has_fast_young_notifiers(struct mm_struct *mm)
+{
+ if (mm_has_notifiers(mm))
+ return __mm_has_fast_young_notifiers(mm);
+ return 0;
+}
+
static inline void
mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
{
@@ -653,6 +662,11 @@ static inline int mmu_notifier_test_young_fast_only(struct mm_struct *mm,
return 0;
}
+static inline bool mm_has_fast_young_notifiers(struct mm_struct *mm)
+{
+ return 0;
+}
+
static inline void
mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
{
@@ -382,6 +382,26 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
return young;
}
+bool __mm_has_fast_young_notifiers(struct mm_struct *mm)
+{
+ struct mmu_notifier *subscription;
+ bool has_fast_aging = false;
+ int id;
+
+ id = srcu_read_lock(&srcu);
+ hlist_for_each_entry_rcu(subscription,
+ &mm->notifier_subscriptions->list, hlist,
+ srcu_read_lock_held(&srcu)) {
+ if (subscription->has_fast_aging) {
+ has_fast_aging = true;
+ break;
+ }
+ }
+ srcu_read_unlock(&srcu, id);
+
+ return has_fast_aging;
+}
+
int __mmu_notifier_clear_young(struct mm_struct *mm,
unsigned long start,
unsigned long end,
@@ -394,6 +414,9 @@ int __mmu_notifier_clear_young(struct mm_struct *mm,
hlist_for_each_entry_rcu(subscription,
&mm->notifier_subscriptions->list, hlist,
srcu_read_lock_held(&srcu)) {
+ if (fast_only && !subscription->has_fast_aging)
+ continue;
+
if (subscription->ops->clear_young)
young |= subscription->ops->clear_young(subscription,
mm, start, end,
@@ -415,6 +438,9 @@ int __mmu_notifier_test_young(struct mm_struct *mm,
hlist_for_each_entry_rcu(subscription,
&mm->notifier_subscriptions->list, hlist,
srcu_read_lock_held(&srcu)) {
+ if (fast_only && !subscription->has_fast_aging)
+ continue;
+
if (subscription->ops->test_young) {
young = subscription->ops->test_young(subscription, mm,
address,
has_fast_aging should be set by subscribers that non-trivially implement fast_only versions of both test_young() and clear_young(). Fast aging must be opt-in. For a subscriber that has not been enlightened with "fast aging", the test/clear_young() will behave identically whether or not fast_only is given. Given that KVM is the only test/clear_young() implementer, we could instead add an equivalent check in KVM, but doing so would incur an indirect function call every time, even if the notifier ends up being a no-op. Add mm_has_fast_young_notifiers() in case a caller wants to know if it should skip many calls to the mmu notifiers that may not be necessary (like MGLRU look-around). Signed-off-by: James Houghton <jthoughton@google.com> --- include/linux/mmu_notifier.h | 14 ++++++++++++++ mm/mmu_notifier.c | 26 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+)