[4/4] aio: allow direct aio poll comletions for keyed wakeups

Commit Message

Christoph Hellwig July 26, 2018, 8:29 a.m. UTC

If we get a keyed wakeup for a aio poll waitqueue and wake can acquire the
ctx_lock without spinning we can just complete the iocb straight from the
wakeup callback to avoid a context switch.

Signed-off-by: Christoph Hellwig <[e-mail]>
---
 fs/aio.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

Patch

diff --git a/fs/aio.c b/fs/aio.c
index cf364d75abe9..ea744092387d 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -1673,11 +1673,25 @@  static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
 		void *key)
 {
 	struct poll_iocb *req = container_of(wait, struct poll_iocb, wait);
+	struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
 	__poll_t mask = key_to_poll(key);
 
 	/* for instances that support it check for an event match first: */
-	if (mask && !(mask & req->events))
-		return 0;
+	if (mask) {
+		if (!(mask & req->events))
+			return 0;
+
+		/* try to complete the iocb inline if we can: */
+		if (spin_trylock(&iocb->ki_ctx->ctx_lock)) {
+			req->done = true;
+			list_del(&iocb->ki_list);
+			spin_unlock(&iocb->ki_ctx->ctx_lock);
+
+			list_del_init(&req->wait.entry);
+			aio_poll_complete(iocb, mask);
+			return 1;
+		}
+	}
 
 	list_del_init(&req->wait.entry);
 	schedule_work(&req->work);