@@ -61,6 +61,17 @@ struct mempolicy {
} wil;
};
+/*
+ * Describes settings of a mempolicy during set/get syscalls and
+ * kernel internal calls to do_set_mempolicy()
+ */
+struct mempolicy_args {
+ unsigned short mode; /* policy mode */
+ unsigned short mode_flags; /* policy mode flags */
+ int home_node; /* mbind: use MPOL_MF_HOME_NODE */
+ nodemask_t *policy_nodes; /* get/set/mbind */
+};
+
/*
* Support for managing mempolicy data objects (clone, copy, destroy)
* The default fast path of a NULL MPOL_DEFAULT policy is always inlined.
@@ -265,10 +265,12 @@ static int mpol_set_nodemask(struct mempolicy *pol,
* This function just creates a new policy, does some check and simple
* initialization. You must invoke mpol_set_nodemask() to set nodes.
*/
-static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
- nodemask_t *nodes)
+static struct mempolicy *mpol_new(struct mempolicy_args *args)
{
struct mempolicy *policy;
+ unsigned short mode = args->mode;
+ unsigned short flags = args->mode_flags;
+ nodemask_t *nodes = args->policy_nodes;
if (mode == MPOL_DEFAULT) {
if (nodes && !nodes_empty(*nodes))
@@ -817,8 +819,7 @@ static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma,
}
/* Set the process memory policy */
-static long do_set_mempolicy(unsigned short mode, unsigned short flags,
- nodemask_t *nodes)
+static long do_set_mempolicy(struct mempolicy_args *args)
{
struct mempolicy *new, *old;
NODEMASK_SCRATCH(scratch);
@@ -827,14 +828,14 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags,
if (!scratch)
return -ENOMEM;
- new = mpol_new(mode, flags, nodes);
+ new = mpol_new(args);
if (IS_ERR(new)) {
ret = PTR_ERR(new);
goto out;
}
task_lock(current);
- ret = mpol_set_nodemask(new, nodes, scratch);
+ ret = mpol_set_nodemask(new, args->policy_nodes, scratch);
if (ret) {
task_unlock(current);
mpol_put(new);
@@ -1232,8 +1233,7 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src,
#endif
static long do_mbind(unsigned long start, unsigned long len,
- unsigned short mode, unsigned short mode_flags,
- nodemask_t *nmask, unsigned long flags)
+ struct mempolicy_args *margs, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma, *prev;
@@ -1253,7 +1253,7 @@ static long do_mbind(unsigned long start, unsigned long len,
if (start & ~PAGE_MASK)
return -EINVAL;
- if (mode == MPOL_DEFAULT)
+ if (margs->mode == MPOL_DEFAULT)
flags &= ~MPOL_MF_STRICT;
len = PAGE_ALIGN(len);
@@ -1264,7 +1264,7 @@ static long do_mbind(unsigned long start, unsigned long len,
if (end == start)
return 0;
- new = mpol_new(mode, mode_flags, nmask);
+ new = mpol_new(margs);
if (IS_ERR(new))
return PTR_ERR(new);
@@ -1281,7 +1281,8 @@ static long do_mbind(unsigned long start, unsigned long len,
NODEMASK_SCRATCH(scratch);
if (scratch) {
mmap_write_lock(mm);
- err = mpol_set_nodemask(new, nmask, scratch);
+ err = mpol_set_nodemask(new, margs->policy_nodes,
+ scratch);
if (err)
mmap_write_unlock(mm);
} else
@@ -1295,7 +1296,7 @@ static long do_mbind(unsigned long start, unsigned long len,
* Lock the VMAs before scanning for pages to migrate,
* to ensure we don't miss a concurrently inserted page.
*/
- nr_failed = queue_pages_range(mm, start, end, nmask,
+ nr_failed = queue_pages_range(mm, start, end, margs->policy_nodes,
flags | MPOL_MF_INVERT | MPOL_MF_WRLOCK, &pagelist);
if (nr_failed < 0) {
@@ -1500,6 +1501,7 @@ static long kernel_mbind(unsigned long start, unsigned long len,
unsigned long mode, const unsigned long __user *nmask,
unsigned long maxnode, unsigned int flags)
{
+ struct mempolicy_args margs;
unsigned short mode_flags;
nodemask_t nodes;
int lmode = mode;
@@ -1514,7 +1516,12 @@ static long kernel_mbind(unsigned long start, unsigned long len,
if (err)
return err;
- return do_mbind(start, len, lmode, mode_flags, &nodes, flags);
+ memset(&margs, 0, sizeof(margs));
+ margs.mode = lmode;
+ margs.mode_flags = mode_flags;
+ margs.policy_nodes = &nodes;
+
+ return do_mbind(start, len, &margs, flags);
}
SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len,
@@ -1595,6 +1602,7 @@ SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
unsigned long maxnode)
{
+ struct mempolicy_args args;
unsigned short mode_flags;
nodemask_t nodes;
int lmode = mode;
@@ -1608,7 +1616,12 @@ static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
if (err)
return err;
- return do_set_mempolicy(lmode, mode_flags, &nodes);
+ memset(&args, 0, sizeof(args));
+ args.mode = lmode;
+ args.mode_flags = mode_flags;
+ args.policy_nodes = &nodes;
+
+ return do_set_mempolicy(&args);
}
SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
@@ -2890,6 +2903,7 @@ static int shared_policy_replace(struct shared_policy *sp, pgoff_t start,
void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
{
int ret;
+ struct mempolicy_args margs;
sp->root = RB_ROOT; /* empty tree == default mempolicy */
rwlock_init(&sp->lock);
@@ -2902,8 +2916,12 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
if (!scratch)
goto put_mpol;
+ memset(&margs, 0, sizeof(margs));
+ margs.mode = mpol->mode;
+ margs.mode_flags = mpol->flags;
+ margs.policy_nodes = &mpol->w.user_nodemask;
/* contextualize the tmpfs mount point mempolicy to this file */
- npol = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
+ npol = mpol_new(&margs);
if (IS_ERR(npol))
goto free_scratch; /* no valid nodemask intersection */
@@ -3011,6 +3029,7 @@ static inline void __init check_numabalancing_enable(void)
void __init numa_policy_init(void)
{
+ struct mempolicy_args args;
nodemask_t interleave_nodes;
unsigned long largest = 0;
int nid, prefer = 0;
@@ -3056,7 +3075,11 @@ void __init numa_policy_init(void)
if (unlikely(nodes_empty(interleave_nodes)))
node_set(prefer, interleave_nodes);
- if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
+ memset(&args, 0, sizeof(args));
+ args.mode = MPOL_INTERLEAVE;
+ args.policy_nodes = &interleave_nodes;
+
+ if (do_set_mempolicy(&args))
pr_err("%s: interleaving failed\n", __func__);
check_numabalancing_enable();
@@ -3065,7 +3088,12 @@ void __init numa_policy_init(void)
/* Reset policy of current process to default */
void numa_default_policy(void)
{
- do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
+ struct mempolicy_args args;
+
+ memset(&args, 0, sizeof(args));
+ args.mode = MPOL_DEFAULT;
+
+ do_set_mempolicy(&args);
}
/*
@@ -3095,6 +3123,7 @@ static const char * const policy_modes[] =
*/
int mpol_parse_str(char *str, struct mempolicy **mpol)
{
+ struct mempolicy_args margs;
struct mempolicy *new = NULL;
unsigned short mode_flags;
nodemask_t nodes;
@@ -3181,7 +3210,11 @@ int mpol_parse_str(char *str, struct mempolicy **mpol)
goto out;
}
- new = mpol_new(mode, mode_flags, &nodes);
+ memset(&margs, 0, sizeof(margs));
+ margs.mode = mode;
+ margs.mode_flags = mode_flags;
+ margs.policy_nodes = &nodes;
+ new = mpol_new(&margs);
if (IS_ERR(new))
goto out;
This patch adds a new kernel structure `struct mempolicy_args`, intended to be used for an extensible get/set_mempolicy interface. This implements the fields required to support the existing syscall interfaces interfaces, but does not expose any user-facing arg structure. mpol_new is refactored to take the argument structure so that future mempolicy extensions can all be managed in the mempolicy constructor. The get_mempolicy and mbind syscalls are refactored to utilize the new argument structure, as are all the callers of mpol_new() and do_set_mempolicy. Signed-off-by: Gregory Price <gregory.price@memverge.com> --- include/linux/mempolicy.h | 11 +++++++ mm/mempolicy.c | 69 +++++++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 18 deletions(-)