[5/8] epoll: simplify and optimize busy loop logic

Commit Message

Soheil Hassas Yeganeh Nov. 6, 2020, 11:16 p.m. UTC

From: Soheil Hassas Yeganeh <[e-mail]>

ep_events_available() is called multiple times around the busy loop
logic, even though the logic is generally not used.
ep_reset_busy_poll_napi_id() is similarly always called, even when
busy loop is not used.

Eliminate ep_reset_busy_poll_napi_id() and inline it inside
ep_busy_loop(). Make ep_busy_loop() return whether there are any
events available after the busy loop.  This will eliminate unnecessary
loads and branches, and simplifies the loop.

Signed-off-by: Soheil Hassas Yeganeh <[e-mail]>
Reviewed-by: Eric Dumazet <[e-mail]>
Reviewed-by: Willem de Bruijn <[e-mail]>
Reviewed-by: Khazhismel Kumykov <[e-mail]>
---
 fs/eventpoll.c | 42 +++++++++++++++++++-----------------------
 1 file changed, 19 insertions(+), 23 deletions(-)

Patch

diff --git a/fs/eventpoll.c b/fs/eventpoll.c
index 5226b0cb1098..28c1d341d2e6 100644
--- a/fs/eventpoll.c
+++ b/fs/eventpoll.c
@@ -391,19 +391,26 @@  static bool ep_busy_loop_end(void *p, unsigned long start_time)
  * busy loop will return if need_resched or ep_events_available.
  *
  * we must do our busy polling with irqs enabled
+ *
+ * Returns whether new events are available after a successful busy loop.
  */
-static void ep_busy_loop(struct eventpoll *ep, int nonblock)
+static bool ep_busy_loop(struct eventpoll *ep, int nonblock)
 {
 	unsigned int napi_id = READ_ONCE(ep->napi_id);
 
-	if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on())
+	if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on()) {
 		napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep);
-}
-
-static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
-{
-	if (ep->napi_id)
+		if (ep_events_available(ep))
+			return true;
+		/*
+		 * Busy poll timed out.  Drop NAPI ID for now, we can add
+		 * it back in when we have moved a socket with a valid NAPI
+		 * ID onto the ready list.
+		 */
 		ep->napi_id = 0;
+		return false;
+	}
+	return false;
 }
 
 /*
@@ -444,12 +451,9 @@  static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
 
 #else
 
-static inline void ep_busy_loop(struct eventpoll *ep, int nonblock)
-{
-}
-
-static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
+static inline bool ep_busy_loop(struct eventpoll *ep, int nonblock)
 {
+	return false;
 }
 
 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
@@ -1857,21 +1861,13 @@  static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
 	}
 
 fetch_events:
-
-	if (!ep_events_available(ep))
-		ep_busy_loop(ep, timed_out);
-
 	eavail = ep_events_available(ep);
+	if (!eavail)
+		eavail = ep_busy_loop(ep, timed_out);
+
 	if (eavail)
 		goto send_events;
 
-	/*
-	 * Busy poll timed out.  Drop NAPI ID for now, we can add
-	 * it back in when we have moved a socket with a valid NAPI
-	 * ID onto the ready list.
-	 */
-	ep_reset_busy_poll_napi_id(ep);
-
 	do {
 		if (signal_pending(current))
 			return -EINTR;