@@ -323,6 +323,11 @@ struct mem_cgroup {
spinlock_t event_list_lock;
#endif /* CONFIG_MEMCG_V1 */
+#ifdef CONFIG_WORKINGSET_REPORT
+ /* memory.workingset.page_age file */
+ struct cgroup_file workingset_page_age_file;
+#endif
+
struct mem_cgroup_per_node *nodeinfo[];
};
@@ -1094,6 +1099,16 @@ static inline void memcg_memory_event_mm(struct mm_struct *mm,
void split_page_memcg(struct page *head, int old_order, int new_order);
+static inline struct cgroup_file *
+mem_cgroup_page_age_file(struct mem_cgroup *memcg)
+{
+#ifdef CONFIG_WORKINGSET_REPORT
+ return &memcg->workingset_page_age_file;
+#else
+ return NULL;
+#endif
+}
+
#else /* CONFIG_MEMCG */
#define MEM_CGROUP_ID_SHIFT 0
@@ -1511,6 +1526,12 @@ void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
static inline void split_page_memcg(struct page *head, int old_order, int new_order)
{
}
+
+static inline struct cgroup_file *
+mem_cgroup_page_age_file(struct mem_cgroup *memcg)
+{
+ return NULL;
+}
#endif /* CONFIG_MEMCG */
/*
@@ -9,6 +9,8 @@ struct mem_cgroup;
struct pglist_data;
struct node;
struct lruvec;
+struct cgroup_file;
+struct wsr_state;
#ifdef CONFIG_WORKINGSET_REPORT
@@ -40,7 +42,10 @@ struct wsr_state {
unsigned long report_threshold;
unsigned long refresh_interval;
- struct kernfs_node *page_age_sys_file;
+ union {
+ struct kernfs_node *page_age_sys_file;
+ struct cgroup_file *page_age_cgroup_file;
+ };
/* breakdown of workingset by page age */
struct mutex page_age_lock;
@@ -60,6 +65,9 @@ void wsr_remove_sysfs(struct node *node);
*/
bool wsr_refresh_report(struct wsr_state *wsr, struct mem_cgroup *root,
struct pglist_data *pgdat);
+
+int wsr_set_refresh_interval(struct wsr_state *wsr,
+ unsigned long refresh_interval);
#else
static inline void wsr_init_lruvec(struct lruvec *lruvec)
{
@@ -79,6 +87,11 @@ static inline void wsr_init_sysfs(struct node *node)
static inline void wsr_remove_sysfs(struct node *node)
{
}
+static inline int wsr_set_refresh_interval(struct wsr_state *wsr,
+ unsigned long refresh_interval)
+{
+ return 0;
+}
#endif /* CONFIG_WORKINGSET_REPORT */
#endif /* _LINUX_WORKINGSET_REPORT_H */
@@ -484,6 +484,8 @@ void set_task_reclaim_state(struct task_struct *task, struct reclaim_state *rs);
* in mm/wsr.c
*/
void notify_workingset(struct mem_cgroup *memcg, struct pglist_data *pgdat);
+int workingset_report_intervals_parse(char *src,
+ struct wsr_report_bins *bins);
#else
static inline void notify_workingset(struct mem_cgroup *memcg,
struct pglist_data *pgdat)
@@ -4348,6 +4348,144 @@ static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf,
return nbytes;
}
+#ifdef CONFIG_WORKINGSET_REPORT
+static int memory_ws_refresh_interval_show(struct seq_file *m, void *v)
+{
+ int nid;
+ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
+
+ for_each_node_state(nid, N_MEMORY) {
+ struct wsr_state *wsr =
+ &mem_cgroup_lruvec(memcg, NODE_DATA(nid))->wsr;
+
+ seq_printf(m, "N%d=%u ", nid,
+ jiffies_to_msecs(READ_ONCE(wsr->refresh_interval)));
+ }
+ seq_putc(m, '\n');
+
+ return 0;
+}
+
+static ssize_t memory_wsr_threshold_parse(char *buf, size_t nbytes,
+ unsigned int *nid_out,
+ unsigned int *msecs)
+{
+ char *node, *threshold;
+ unsigned int nid;
+ int err;
+
+ buf = strstrip(buf);
+ threshold = buf;
+ node = strsep(&threshold, "=");
+
+ if (*node != 'N')
+ return -EINVAL;
+
+ err = kstrtouint(node + 1, 0, &nid);
+ if (err)
+ return err;
+
+ if (nid >= nr_node_ids || !node_state(nid, N_MEMORY))
+ return -EINVAL;
+
+ err = kstrtouint(threshold, 0, msecs);
+ if (err)
+ return err;
+
+ *nid_out = nid;
+
+ return nbytes;
+}
+
+static ssize_t memory_ws_refresh_interval_write(struct kernfs_open_file *of,
+ char *buf, size_t nbytes,
+ loff_t off)
+{
+ unsigned int nid, msecs;
+ struct wsr_state *wsr;
+ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
+ ssize_t ret = memory_wsr_threshold_parse(buf, nbytes, &nid, &msecs);
+
+ if (ret < 0)
+ return ret;
+
+ wsr = &mem_cgroup_lruvec(memcg, NODE_DATA(nid))->wsr;
+
+ ret = wsr_set_refresh_interval(wsr, msecs);
+ return ret ?: nbytes;
+}
+
+static int memory_ws_report_threshold_show(struct seq_file *m, void *v)
+{
+ int nid;
+ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
+
+ for_each_node_state(nid, N_MEMORY) {
+ struct wsr_state *wsr =
+ &mem_cgroup_lruvec(memcg, NODE_DATA(nid))->wsr;
+
+ seq_printf(m, "N%d=%u ", nid,
+ jiffies_to_msecs(READ_ONCE(wsr->report_threshold)));
+ }
+ seq_putc(m, '\n');
+
+ return 0;
+}
+
+static ssize_t memory_ws_report_threshold_write(struct kernfs_open_file *of,
+ char *buf, size_t nbytes,
+ loff_t off)
+{
+ unsigned int nid, msecs;
+ struct wsr_state *wsr;
+ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
+ ssize_t ret = memory_wsr_threshold_parse(buf, nbytes, &nid, &msecs);
+
+ if (ret < 0)
+ return ret;
+
+ wsr = &mem_cgroup_lruvec(memcg, NODE_DATA(nid))->wsr;
+ WRITE_ONCE(wsr->report_threshold, msecs_to_jiffies(msecs));
+ return ret;
+}
+
+static int memory_ws_page_age_show(struct seq_file *m, void *v)
+{
+ int nid;
+ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
+
+ for_each_node_state(nid, N_MEMORY) {
+ struct wsr_state *wsr =
+ &mem_cgroup_lruvec(memcg, NODE_DATA(nid))->wsr;
+ struct wsr_report_bin *bin;
+
+ if (!READ_ONCE(wsr->page_age))
+ continue;
+
+ wsr_refresh_report(wsr, memcg, NODE_DATA(nid));
+ mutex_lock(&wsr->page_age_lock);
+ if (!wsr->page_age)
+ goto unlock;
+ seq_printf(m, "N%d\n", nid);
+ for (bin = wsr->page_age->bins;
+ bin->idle_age != WORKINGSET_INTERVAL_MAX; bin++)
+ seq_printf(m, "%u anon=%lu file=%lu\n",
+ jiffies_to_msecs(bin->idle_age),
+ bin->nr_pages[0] * PAGE_SIZE,
+ bin->nr_pages[1] * PAGE_SIZE);
+
+ seq_printf(m, "%lu anon=%lu file=%lu\n", WORKINGSET_INTERVAL_MAX,
+ bin->nr_pages[0] * PAGE_SIZE,
+ bin->nr_pages[1] * PAGE_SIZE);
+
+unlock:
+ mutex_unlock(&wsr->page_age_lock);
+ }
+
+ return 0;
+}
+#endif
+
static struct cftype memory_files[] = {
{
.name = "current",
@@ -4419,7 +4557,27 @@ static struct cftype memory_files[] = {
.flags = CFTYPE_NS_DELEGATABLE,
.write = memory_reclaim,
},
- { } /* terminate */
+#ifdef CONFIG_WORKINGSET_REPORT
+ {
+ .name = "workingset.refresh_interval",
+ .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
+ .seq_show = memory_ws_refresh_interval_show,
+ .write = memory_ws_refresh_interval_write,
+ },
+ {
+ .name = "workingset.report_threshold",
+ .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
+ .seq_show = memory_ws_report_threshold_show,
+ .write = memory_ws_report_threshold_write,
+ },
+ {
+ .name = "workingset.page_age",
+ .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
+ .file_offset = offsetof(struct mem_cgroup, workingset_page_age_file),
+ .seq_show = memory_ws_page_age_show,
+ },
+#endif
+ {} /* terminate */
};
struct cgroup_subsys memory_cgrp_subsys = {
@@ -37,9 +37,12 @@ void wsr_destroy_pgdat(struct pglist_data *pgdat)
void wsr_init_lruvec(struct lruvec *lruvec)
{
struct wsr_state *wsr = &lruvec->wsr;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
memset(wsr, 0, sizeof(*wsr));
mutex_init(&wsr->page_age_lock);
+ if (memcg && !mem_cgroup_is_root(memcg))
+ wsr->page_age_cgroup_file = mem_cgroup_page_age_file(memcg);
}
void wsr_destroy_lruvec(struct lruvec *lruvec)
@@ -51,8 +54,8 @@ void wsr_destroy_lruvec(struct lruvec *lruvec)
memset(wsr, 0, sizeof(*wsr));
}
-static int workingset_report_intervals_parse(char *src,
- struct wsr_report_bins *bins)
+int workingset_report_intervals_parse(char *src,
+ struct wsr_report_bins *bins)
{
int err = 0, i = 0;
char *cur, *next = strim(src);
@@ -356,20 +359,14 @@ static ssize_t refresh_interval_show(struct kobject *kobj,
return sysfs_emit(buf, "%u\n", jiffies_to_msecs(interval));
}
-static ssize_t refresh_interval_store(struct kobject *kobj,
- struct kobj_attribute *attr,
- const char *buf, size_t len)
+int wsr_set_refresh_interval(struct wsr_state *wsr,
+ unsigned long refresh_interval)
{
- unsigned int interval;
- int err;
- struct wsr_state *wsr = kobj_to_wsr(kobj);
-
- err = kstrtouint(buf, 0, &interval);
- if (err)
- return err;
+ int err = 0;
+ unsigned long old_interval = 0;
mutex_lock(&wsr->page_age_lock);
- if (interval && !wsr->page_age) {
+ if (refresh_interval && !wsr->page_age) {
struct wsr_page_age_histo *page_age =
kzalloc(sizeof(struct wsr_page_age_histo), GFP_KERNEL);
@@ -377,16 +374,34 @@ static ssize_t refresh_interval_store(struct kobject *kobj,
err = -ENOMEM;
goto unlock;
}
+ page_age->bins[0].idle_age = WORKINGSET_INTERVAL_MAX;
wsr->page_age = page_age;
}
- if (!interval && wsr->page_age) {
+ if (!refresh_interval && wsr->page_age) {
kfree(wsr->page_age);
wsr->page_age = NULL;
}
- WRITE_ONCE(wsr->refresh_interval, msecs_to_jiffies(interval));
+ old_interval = READ_ONCE(wsr->refresh_interval);
+ WRITE_ONCE(wsr->refresh_interval, msecs_to_jiffies(refresh_interval));
unlock:
mutex_unlock(&wsr->page_age_lock);
+ return err;
+}
+
+static ssize_t refresh_interval_store(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t len)
+{
+ unsigned int interval;
+ int err;
+ struct wsr_state *wsr = kobj_to_wsr(kobj);
+
+ err = kstrtouint(buf, 0, &interval);
+ if (err)
+ return err;
+
+ err = wsr_set_refresh_interval(wsr, interval);
return err ?: len;
}
@@ -551,5 +566,8 @@ void notify_workingset(struct mem_cgroup *memcg, struct pglist_data *pgdat)
{
struct wsr_state *wsr = &mem_cgroup_lruvec(memcg, pgdat)->wsr;
- kernfs_notify(wsr->page_age_sys_file);
+ if (mem_cgroup_is_root(memcg))
+ kernfs_notify(wsr->page_age_sys_file);
+ else
+ cgroup_file_notify(wsr->page_age_cgroup_file);
}
Break down the system-wide workingset report into per-memcg reports, which aggregages its children hierarchically. The per-node workingset reporting histograms and refresh/report threshold files are presented as memcg files, showing a report containing all the nodes. The per-node page age interval is configurable in sysfs and not available per-memcg, while the refresh interval and report threshold are configured per-memcg. Memcg interface: /sys/fs/cgroup/.../memory.workingset.page_age The memcg equivalent of the sysfs workingset page age histogram breaks down the workingset of this memcg and its children into page age intervals. Each node is prefixed with a node header and a newline. Non-proactive direct reclaim on this memcg can also wake up userspace agents that are waiting on this file. e.g. N0 1000 anon=0 file=0 2000 anon=0 file=0 3000 anon=0 file=0 4000 anon=0 file=0 5000 anon=0 file=0 18446744073709551615 anon=0 file=0 /sys/fs/cgroup/.../memory.workingset.refresh_interval The memcg equivalent of the sysfs refresh interval. A per-node number of how much time a page age histogram is valid for, in milliseconds. e.g. echo N0=2000 > memory.workingset.refresh_interval /sys/fs/cgroup/.../memory.workingset.report_threshold The memcg equivalent of the sysfs report threshold. A per-node number of how often userspace agent waiting on the page age histogram can be woken up, in milliseconds. e.g. echo N0=1000 > memory.workingset.report_threshold Signed-off-by: Yuanchu Xie <yuanchu@google.com> --- include/linux/memcontrol.h | 21 ++++ include/linux/workingset_report.h | 15 ++- mm/internal.h | 2 + mm/memcontrol.c | 160 +++++++++++++++++++++++++++++- mm/workingset_report.c | 50 +++++++--- 5 files changed, 230 insertions(+), 18 deletions(-)