@@ -4789,6 +4789,17 @@ struct loc_track {
struct location *loc;
};
+enum slub_list_field { PARTIAL_LIST, FULL_LIST };
+
+struct slab_debug_private {
+ struct inode *inode;
+ struct kmem_cache_node *node;
+ unsigned long nid;
+ long slabs_remaining;
+ enum slub_list_field field;
+ unsigned long *map;
+};
+
static struct dentry *slab_debugfs_root;
struct loc_track t = { 0, 0, NULL };
@@ -5809,6 +5820,216 @@ static int debugfs_slab_alias(struct kmem_cache *s, const char *name)
return 0;
}
+static struct kmem_cache_node *find_node(struct kmem_cache *s, unsigned long *nid)
+{
+ struct kmem_cache_node *node = NULL;
+
+ while (*nid < nr_node_ids) {
+ node = s->node[*nid];
+ ++*nid;
+ if (!node || !atomic_long_read(&node->nr_slabs))
+ node = NULL;
+ else
+ break;
+ }
+ return node;
+}
+
+static bool next_page_new_node(struct slab_debug_private *priv)
+{
+ struct kmem_cache_node *node;
+ struct kmem_cache *s = priv->inode->i_private;
+
+ node = find_node(s, &priv->nid);
+
+ if (!node)
+ return false;
+
+ priv->node = node;
+
+ if (node->nr_partial > 0) {
+ priv->field = PARTIAL_LIST;
+ priv->slabs_remaining = node->nr_partial;
+ } else if (atomic_long_read(&node->nr_slabs) > 0) {
+ priv->field = FULL_LIST;
+ priv->slabs_remaining = atomic_long_read(&node->nr_slabs);
+ }
+
+ return priv->slabs_remaining;
+}
+
+static struct page *next_page(struct slab_debug_private *priv)
+{
+ struct page *page = NULL;
+ struct kmem_cache *s = priv->inode->i_private;
+ struct kmem_cache_node *node;
+ unsigned long flags;
+
+redo:
+ node = priv->node;
+ if (priv->slabs_remaining > 0) {
+ struct list_head *head;
+ void *p, *addr;
+
+ --priv->slabs_remaining;
+
+ if (priv->field == PARTIAL_LIST)
+ head = &node->partial;
+ else
+ head = &node->full;
+
+ spin_lock_irqsave(&node->list_lock, flags);
+ page = list_first_entry(head, struct page, slab_list);
+ if (page) {
+ get_page(page);
+ slab_lock(page);
+ addr = page_address(page);
+ bitmap_zero(priv->map, page->objects);
+
+ for (p = page->freelist; p; p = get_freepointer(s, p))
+ set_bit(__obj_to_index(s, addr, p), priv->map);
+ slab_unlock(page);
+ }
+ list_rotate_left(head);
+ spin_unlock_irqrestore(&node->list_lock, flags);
+
+ } else if ((priv->field == PARTIAL_LIST)
+ && (atomic_long_read(&node->nr_slabs) != node->nr_partial)) {
+
+ priv->field = FULL_LIST;
+ priv->slabs_remaining = atomic_long_read(&node->nr_slabs) - node->nr_partial;
+
+ goto redo;
+ } else {
+ if (next_page_new_node(priv))
+ goto redo;
+ }
+
+ return page;
+}
+
+static int debugfs_all_objects_show(struct seq_file *seq, void *v)
+{
+ struct slab_debug_private *priv = seq->private;
+ struct kmem_cache *s = priv->inode->i_private;
+ struct page *page = v;
+ void *addr = page_address(page);
+ void *p;
+ unsigned long *map = priv->map;
+ struct track *track;
+ depot_stack_handle_t handle;
+ unsigned long *entries;
+ unsigned int nr_entries, j;
+
+ for_each_object(p, s, addr, page->objects) {
+ seq_printf(seq, "Object: %pK ", p);
+ if (!test_bit(__obj_to_index(s, addr, p), map))
+ seq_puts(seq, "allocated\n");
+ else
+ seq_puts(seq, "free\n");
+
+ track = get_track(s, p, TRACK_ALLOC);
+ seq_printf(seq, "Last allocated: %pS age=%ld pid=%d cpu=%u\n",
+ (void *)track->addr, jiffies - track->when, track->pid, track->cpu);
+
+#ifdef CONFIG_STACKDEPOT
+ handle = READ_ONCE(track->handle);
+ if (handle) {
+ nr_entries = stack_depot_fetch(handle, &entries);
+ for (j = 0; j < nr_entries; j++)
+ seq_printf(seq, "\t%pS\n", (void *)entries[j]);
+ }
+#endif
+
+ track = get_track(s, p, TRACK_FREE);
+ seq_printf(seq, "Last free: %pS age=%ld pid=%d cpu=%u\n",
+ (void *)track->addr, jiffies - track->when, track->pid, track->cpu);
+
+#ifdef CONFIG_STACKDEPOT
+ handle = READ_ONCE(track->handle);
+ if (handle) {
+ nr_entries = stack_depot_fetch(handle, &entries);
+ for (j = 0; j < nr_entries; j++)
+ seq_printf(seq, "\t%pS\n", (void *)entries[j]);
+ }
+#endif
+ seq_puts(seq, "\n");
+ }
+ return 0;
+}
+
+static void *debugfs_all_objects_start(struct seq_file *m, loff_t *ppos)
+{
+ struct slab_debug_private *priv = m->private;
+ struct kmem_cache *s = priv->inode->i_private;
+ struct page *page;
+
+ priv->map = kmalloc(BITS_TO_LONGS(MAX_OBJS_PER_PAGE), GFP_KERNEL);
+
+ if (!priv->map)
+ return NULL;
+
+ if (!(s->flags & SLAB_STORE_USER))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ page = next_page(priv);
+ return page;
+}
+
+static void *debugfs_all_objects_next(struct seq_file *m, void *v, loff_t *ppos)
+{
+ struct slab_debug_private *priv = m->private;
+ struct page *page;
+
+ if (v)
+ put_page(v);
+
+ ++*ppos;
+ page = next_page(priv);
+
+ return page;
+}
+
+static void debugfs_all_objects_stop(struct seq_file *m, void *v)
+{
+ struct slab_debug_private *priv = m->private;
+ struct kmem_cache *s = priv->inode->i_private;
+
+ kfree(priv->map);
+
+ if (v && (s->flags & SLAB_STORE_USER))
+ put_page(v);
+}
+
+static const struct seq_operations debugfs_all_objects_ops = {
+ .start = debugfs_all_objects_start,
+ .next = debugfs_all_objects_next,
+ .stop = debugfs_all_objects_stop,
+ .show = debugfs_all_objects_show
+};
+
+static int debugfs_all_objects_open(struct inode *inode, struct file *file)
+{
+ struct slab_debug_private *priv = __seq_open_private(file,
+ &debugfs_all_objects_ops, sizeof(struct slab_debug_private));
+
+ if (!priv)
+ return -ENOMEM;
+
+ priv->inode = inode;
+ priv->nid = 0;
+ priv->field = FULL_LIST;
+
+ return 0;
+}
+
+static const struct file_operations debugfs_all_objects_fops = {
+ .open = debugfs_all_objects_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release_private,
+};
+
static int slab_debugfs_show(struct seq_file *seq, void *v)
{
struct location *l;
@@ -6018,6 +6239,10 @@ static void debugfs_slab_add(struct kmem_cache *s)
debugfs_create_file("free_traces", 0400,
slab_cache_dir, s, &slab_debugfs_fops);
+ debugfs_create_file("all_objects", 0400,
+ slab_cache_dir, s, &debugfs_all_objects_fops);
+
+
if (!unmergeable)
/* Setup first alias */
debugfs_slab_alias(s, s->name);