@@ -44,6 +44,7 @@
#include <linux/sched/sysctl.h>
#include <linux/memory-tiers.h>
#include <linux/pagewalk.h>
+#include <linux/debugfs.h>
#include <asm/tlbflush.h>
@@ -791,6 +792,126 @@ int migrate_folio(struct address_space *mapping, struct folio *dst,
EXPORT_SYMBOL(migrate_folio);
#ifdef CONFIG_BUFFER_HEAD
+
+static const char * const bh_routine_names[] = {
+ "buffer_migrate_folio",
+ "buffer_migrate_folio_norefs",
+};
+
+#define BH_STATS(X) \
+ X(bh_migrate_folio, 0, "calls") \
+ X(bh_migrate_folio_success, 0, "success") \
+ X(bh_migrate_folio_fails, 0, "fails") \
+ X(bh_migrate_folio_norefs, 1, "calls") \
+ X(bh_migrate_folio_norefs_success, 1, "success") \
+ X(bh_migrate_folio_norefs_fails, 1, "fails") \
+ X(bh_migrate_folio_norefs_meta, 1, "jbd-meta") \
+ X(bh_migrate_folio_norefs_nohead_success, 1, "no-head-success") \
+ X(bh_migrate_folio_norefs_nohead_fails, 1, "no-head-fails") \
+ X(bh_migrate_folio_norefs_invalid, 1, "invalid") \
+ X(bh_migrate_folio_norefs_valid, 1, "valid") \
+ X(bh_migrate_folio_norefs_valid_success, 1, "valid-success") \
+ X(bh_migrate_folio_norefs_valid_fails, 1, "valid-fails")
+
+
+#define DECLARE_STAT(name, routine_idx, meaning) static atomic_long_t name;
+BH_STATS(DECLARE_STAT)
+
+#define BH_STAT_PTR(name, routine_idx, meaning) &name,
+static atomic_long_t * const bh_stat_array[] = {
+ BH_STATS(BH_STAT_PTR)
+};
+
+#define BH_STAT_ROUTINE_IDX(name, routine_idx, meaning) routine_idx,
+static const int bh_stat_routine_index[] = {
+ BH_STATS(BH_STAT_ROUTINE_IDX)
+};
+
+#define BH_STAT_MEANING(name, routine_idx, meaning) meaning,
+static const char * const bh_stat_meanings[] = {
+ BH_STATS(BH_STAT_MEANING)
+};
+
+#define NUM_BH_STATS ARRAY_SIZE(bh_stat_array)
+
+static ssize_t read_file_bh_migrate_stats(struct file *file,
+ char __user *user_buf,
+ size_t count, loff_t *ppos)
+{
+ char *buf;
+ unsigned int i, len = 0, size = NUM_BH_STATS * 128;
+ int ret, last_routine = -1;
+ unsigned long total, success, rate;
+
+ BUILD_BUG_ON(ARRAY_SIZE(bh_stat_array) != ARRAY_SIZE(bh_stat_meanings));
+
+ buf = kzalloc(size, GFP_KERNEL);
+ if (buf == NULL)
+ return -ENOMEM;
+
+ for (i = 0; i < NUM_BH_STATS; i++) {
+ int routine_idx = bh_stat_routine_index[i];
+
+ if (routine_idx != last_routine) {
+ len += scnprintf(buf + len, size - len, "\n[%s]\n",
+ bh_routine_names[routine_idx]);
+ last_routine = routine_idx;
+ }
+
+ len += scnprintf(buf + len, size - len, "%25s\t%lu\n",
+ bh_stat_meanings[i],
+ atomic_long_read(bh_stat_array[i]));
+
+ }
+
+ len += scnprintf(buf + len, size - len, "\nSuccess ratios:\n");
+
+ total = atomic_long_read(&bh_migrate_folio);
+ success = atomic_long_read(&bh_migrate_folio_success);
+ rate = total ? (success * 100) / total : 0;
+ len += scnprintf(buf + len, size - len,
+ "%s: %lu%% success (%lu/%lu)\n",
+ "buffer_migrate_folio", rate, success, total);
+
+ total = atomic_long_read(&bh_migrate_folio_norefs);
+ success = atomic_long_read(&bh_migrate_folio_norefs_success);
+ rate = total ? (success * 100) / total : 0;
+ len += scnprintf(buf + len, size - len,
+ "%s: %lu%% success (%lu/%lu)\n",
+ "buffer_migrate_folio_norefs", rate, success, total);
+
+ ret = simple_read_from_buffer(user_buf, count, ppos, buf, len);
+ kfree(buf);
+ return ret;
+}
+
+static const struct file_operations fops_bh_migrate_stats = {
+ .read = read_file_bh_migrate_stats,
+ .open = simple_open,
+ .owner = THIS_MODULE,
+ .llseek = default_llseek,
+};
+
+static void mm_migrate_bh_init(struct dentry *migrate_debug_root)
+{
+ struct dentry *parent_dirs[ARRAY_SIZE(bh_routine_names)] = { NULL };
+ struct dentry *root = debugfs_create_dir("bh", migrate_debug_root);
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(bh_routine_names); i++)
+ parent_dirs[i] = debugfs_create_dir(bh_routine_names[i], root);
+
+ for (i = 0; i < NUM_BH_STATS; i++) {
+ int routine = bh_stat_routine_index[i];
+ debugfs_create_ulong(bh_stat_meanings[i], 0400,
+ parent_dirs[routine],
+ (unsigned long *)
+ &bh_stat_array[i]->counter);
+ }
+
+ debugfs_create_file("stats", 0400, root, root, &fops_bh_migrate_stats);
+}
+
/* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head,
enum migrate_mode mode)
@@ -833,16 +954,26 @@ static int __buffer_migrate_folio(struct address_space *mapping,
int expected_count;
head = folio_buffers(src);
- if (!head)
- return migrate_folio(mapping, dst, src, mode);
+ if (!head) {
+ rc = migrate_folio(mapping, dst, src, mode);
+ if (check_refs) {
+ if (rc == 0)
+ atomic_long_inc(&bh_migrate_folio_norefs_nohead_success);
+ else
+ atomic_long_inc(&bh_migrate_folio_norefs_nohead_fails);
+ }
+ return rc;
+ }
/* Check whether page does not have extra refs before we do more work */
expected_count = folio_expected_refs(mapping, src);
if (folio_ref_count(src) != expected_count)
return -EAGAIN;
- if (buffer_meta(head))
+ if (buffer_meta(head)) {
+ atomic_long_inc(&bh_migrate_folio_norefs_meta);
return -EAGAIN;
+ }
if (!buffer_migrate_lock_buffers(head, mode))
return -EAGAIN;
@@ -868,17 +999,23 @@ static int __buffer_migrate_folio(struct address_space *mapping,
if (busy) {
if (invalidated) {
rc = -EAGAIN;
+ atomic_long_inc(&bh_migrate_folio_norefs_invalid);
goto unlock_buffers;
}
invalidate_bh_lrus();
invalidated = true;
goto recheck_buffers;
}
+ atomic_long_inc(&bh_migrate_folio_norefs_valid);
}
rc = filemap_migrate_folio(mapping, dst, src, mode);
- if (rc != MIGRATEPAGE_SUCCESS)
+ if (rc != MIGRATEPAGE_SUCCESS) {
+ if (check_refs)
+ atomic_long_inc(&bh_migrate_folio_norefs_valid_fails);
goto unlock_buffers;
+ } else if (check_refs)
+ atomic_long_inc(&bh_migrate_folio_norefs_valid_success);
bh = head;
do {
@@ -915,7 +1052,16 @@ static int __buffer_migrate_folio(struct address_space *mapping,
int buffer_migrate_folio(struct address_space *mapping,
struct folio *dst, struct folio *src, enum migrate_mode mode)
{
- return __buffer_migrate_folio(mapping, dst, src, mode, false);
+ int ret;
+ atomic_long_inc(&bh_migrate_folio);
+
+ ret = __buffer_migrate_folio(mapping, dst, src, mode, false);
+ if (ret == 0)
+ atomic_long_inc(&bh_migrate_folio_success);
+ else
+ atomic_long_inc(&bh_migrate_folio_fails);
+
+ return ret;
}
EXPORT_SYMBOL(buffer_migrate_folio);
@@ -936,9 +1082,21 @@ EXPORT_SYMBOL(buffer_migrate_folio);
int buffer_migrate_folio_norefs(struct address_space *mapping,
struct folio *dst, struct folio *src, enum migrate_mode mode)
{
- return __buffer_migrate_folio(mapping, dst, src, mode, true);
+ int ret;
+
+ atomic_long_inc(&bh_migrate_folio_norefs);
+
+ ret = __buffer_migrate_folio(mapping, dst, src, mode, true);
+ if (ret == 0)
+ atomic_long_inc(&bh_migrate_folio_norefs_success);
+ else
+ atomic_long_inc(&bh_migrate_folio_norefs_fails);
+
+ return ret;
}
EXPORT_SYMBOL_GPL(buffer_migrate_folio_norefs);
+#else
+static inline void mm_migrate_bh_init(struct dentry *migrate_debug_root) { }
#endif /* CONFIG_BUFFER_HEAD */
int filemap_migrate_folio(struct address_space *mapping,
@@ -2737,3 +2895,17 @@ int migrate_misplaced_folio(struct folio *folio, int node)
}
#endif /* CONFIG_NUMA_BALANCING */
#endif /* CONFIG_NUMA */
+
+static __init int mm_migrate_debugfs_init(void)
+{
+ struct dentry *mm_debug_root;
+ struct dentry *migrate_debug_root;
+
+ mm_debug_root = debugfs_create_dir("mm", NULL);
+ migrate_debug_root = debugfs_create_dir("migrate", mm_debug_root);
+
+ mm_migrate_bh_init(migrate_debug_root);
+
+ return 0;
+}
+fs_initcall(mm_migrate_debugfs_init);
If you are working on enhancing folio migration it is easy to not be certain on improvements. This debugfs interface enables you to evaluate gains on improvements on buffer-head folio migration. This can easily tell you *why* folio migration might fail, for example, here is the output of a generic/750 run for 18 hours: root@e3-ext4-2k ~ # cat /sys/kernel/debug/mm/migrate/bh/stats [buffer_migrate_folio] calls 50160811 success 50047572 fails 113239 [buffer_migrate_folio_norefs] calls 23577082468 success 2939858 fails 23574142610 jbd-meta 23425956714 no-head-success 102 no-head-fails 0 invalid 147919982 valid 2939881 valid-success 2939756 valid-fails 125 Success ratios: buffer_migrate_folio: 99% success (50047572/50160811) buffer_migrate_folio_norefs: 0% success (2939858/23577082468) Signed-off-by: Luis Chamberlain <mcgrof@kernel.org> --- mm/migrate.c | 184 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 178 insertions(+), 6 deletions(-)