@@ -865,6 +865,23 @@ static void epi_rcu_free(struct rcu_head *head)
kmem_cache_free(epi_cache, epi);
}
+static inline int ep_get_bit(struct eventpoll *ep)
+{
+ bool was_set;
+ int bit;
+
+ lockdep_assert_held(&ep->mtx);
+
+ bit = find_first_zero_bit(ep->items_bm, ep->max_items_nr);
+ if (bit >= ep->max_items_nr)
+ return -ENOSPC;
+
+ was_set = test_and_set_bit(bit, ep->items_bm);
+ WARN_ON(was_set);
+
+ return bit;
+}
+
#define atomic_set_unless_zero(ptr, flags) \
({ \
typeof(ptr) _ptr = (ptr); \
@@ -1875,6 +1892,7 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
struct epitem *epi;
struct ep_pqueue epq;
+ lockdep_assert_held(&ep->mtx);
lockdep_assert_irqs_enabled();
user_watches = atomic_long_read(&ep->user->epoll_watches);
@@ -1901,6 +1919,29 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
RCU_INIT_POINTER(epi->ws, NULL);
}
+ if (ep_polled_by_user(ep)) {
+ struct epoll_uitem *uitem;
+ int bit;
+
+ bit = ep_get_bit(ep);
+ if (unlikely(bit < 0)) {
+ error = bit;
+ goto error_get_bit;
+ }
+ epi->bit = bit;
+
+ /*
+ * Now fill-in user item. Do not touch ready_events, since
+ * it can be EPOLLREMOVED (has been set by previous user
+ * item), thus user index entry can be not yet consumed
+ * by userspace. See ep_remove_user_item() and
+ * ep_add_event_to_uring() for details.
+ */
+ uitem = &ep->user_header->items[epi->bit];
+ uitem->events = event->events;
+ uitem->data = event->data;
+ }
+
/* Initialize the poll table using the queue callback */
epq.epi = epi;
init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
@@ -1945,16 +1986,23 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
/* record NAPI ID of new item if present */
ep_set_busy_poll_napi_id(epi);
- /* If the file is already "ready" we drop it inside the ready list */
- if (revents && !ep_is_linked(epi)) {
- list_add_tail(&epi->rdllink, &ep->rdllist);
- ep_pm_stay_awake(epi);
+ if (revents) {
+ bool added = false;
- /* Notify waiting tasks that events are available */
- if (waitqueue_active(&ep->wq))
- wake_up(&ep->wq);
- if (waitqueue_active(&ep->poll_wait))
- pwake++;
+ if (ep_polled_by_user(ep)) {
+ added = ep_add_event_to_uring(epi, revents);
+ } else if (!ep_is_linked(epi)) {
+ list_add_tail(&epi->rdllink, &ep->rdllist);
+ ep_pm_stay_awake(epi);
+ added = true;
+ }
+ if (added) {
+ /* Notify waiting tasks that events are available */
+ if (waitqueue_active(&ep->wq))
+ wake_up(&ep->wq);
+ if (waitqueue_active(&ep->poll_wait))
+ pwake++;
+ }
}
write_unlock_irq(&ep->lock);
@@ -1983,11 +2031,16 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
* list, since that is used/cleaned only inside a section bound by "mtx".
* And ep_insert() is called with "mtx" held.
*/
- write_lock_irq(&ep->lock);
- if (ep_is_linked(epi))
- list_del_init(&epi->rdllink);
- write_unlock_irq(&ep->lock);
+ if (ep_polled_by_user(ep)) {
+ ep_remove_user_item(epi);
+ } else {
+ write_lock_irq(&ep->lock);
+ if (ep_is_linked(epi))
+ list_del_init(&epi->rdllink);
+ write_unlock_irq(&ep->lock);
+ }
+error_get_bit:
wakeup_source_unregister(ep_wakeup_source(epi));
error_create_wakeup_source:
When epfd is polled by userspace and new item is inserted new bit should be get from a bitmap and then user item is set accordingly. Signed-off-by: Roman Penyaev <rpenyaev@suse.de> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: linux-fsdevel@vger.kernel.org Cc: linux-kernel@vger.kernel.org