@@ -250,6 +250,13 @@ struct aio_submit_state {
struct list_head req_list;
unsigned int req_count;
+ /*
+ * aio_kiocb alloc cache
+ */
+ void *iocbs[AIO_IOPOLL_BATCH];
+ unsigned int free_iocbs;
+ unsigned int cur_iocb;
+
/*
* File reference cache
*/
@@ -1105,15 +1112,34 @@ static void aio_iocb_init(struct kioctx *ctx, struct aio_kiocb *req)
* Allocate a slot for an aio request.
* Returns NULL if no requests are free.
*/
-static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
+static struct aio_kiocb *aio_get_req(struct kioctx *ctx,
+ struct aio_submit_state *state)
{
struct aio_kiocb *req;
- req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL);
- if (unlikely(!req))
- return NULL;
+ if (!state)
+ req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL);
+ else if (!state->free_iocbs) {
+ size_t size;
+ int ret;
+
+ size = min_t(size_t, state->ios_left, ARRAY_SIZE(state->iocbs));
+ ret = kmem_cache_alloc_bulk(kiocb_cachep, GFP_KERNEL, size,
+ state->iocbs);
+ if (ret <= 0)
+ return ERR_PTR(-ENOMEM);
+ state->free_iocbs = ret - 1;
+ state->cur_iocb = 1;
+ req = state->iocbs[0];
+ } else {
+ req = state->iocbs[state->cur_iocb];
+ state->free_iocbs--;
+ state->cur_iocb++;
+ }
+
+ if (req)
+ aio_iocb_init(ctx, req);
- aio_iocb_init(ctx, req);
return req;
}
@@ -2293,7 +2319,7 @@ static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb,
return -EAGAIN;
ret = -EAGAIN;
- req = aio_get_req(ctx);
+ req = aio_get_req(ctx, state);
if (unlikely(!req))
goto out_put_reqs_available;
@@ -2414,6 +2440,9 @@ static void aio_submit_state_end(struct aio_submit_state *state)
if (!list_empty(&state->req_list))
aio_flush_state_reqs(state->ctx, state);
aio_file_put(state, NULL);
+ if (state->free_iocbs)
+ kmem_cache_free_bulk(kiocb_cachep, state->free_iocbs,
+ &state->iocbs[state->cur_iocb]);
}
/*
@@ -2425,6 +2454,7 @@ static void aio_submit_state_start(struct aio_submit_state *state,
state->ctx = ctx;
INIT_LIST_HEAD(&state->req_list);
state->req_count = 0;
+ state->free_iocbs = 0;
state->file = NULL;
state->ios_left = max_ios;
#ifdef CONFIG_BLOCK
Similarly to how we use the state->ios_left to know how many references to get to a file, we can use it to allocate the aio_kiocb's we need in bulk. Signed-off-by: Jens Axboe <axboe@kernel.dk> --- fs/aio.c | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-)