@@ -34,6 +34,11 @@ struct devx_async_data {
struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
};
+struct devx_async_event_data {
+ struct list_head list; /* headed in ev_queue->event_list */
+ struct mlx5_ib_uapi_devx_async_event_hdr hdr;
+};
+
/* first level XA value data structure */
struct devx_event {
struct xarray object_ids; /* second XA level, Key = object id */
@@ -54,6 +59,7 @@ struct devx_event_subscription {
* devx_obj_event->obj_sub_list
*/
struct list_head obj_list; /* headed in devx_object */
+ struct list_head event_list; /* headed in ev_queue->event_list */
u32 xa_key_level1;
u32 xa_key_level2;
@@ -70,6 +76,7 @@ struct devx_async_event_queue {
struct list_head event_list;
atomic_t bytes_in_use;
u8 is_destroyed:1;
+ u8 is_overflow_err:1;
u32 flags;
};
@@ -293,6 +300,46 @@ static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
}
}
+/* Any future affiliated event should have a fixed header to get the obj
+ * type and id including events on legacy objects.
+ */
+static u32 get_affiliated_event_obj_id(struct mlx5_eqe *eqe)
+{
+ u32 obj_id = MLX5_GET(affiliated_event_header, eqe, obj_id);
+
+ return obj_id;
+}
+
+static u16 get_affiliated_event_obj_type(struct mlx5_eqe *eqe)
+{
+ u16 obj_type = MLX5_GET(affiliated_event_header, eqe, obj_type);
+
+ return obj_type;
+}
+
+static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
+{
+ switch (event_type) {
+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
+ case MLX5_EVENT_TYPE_PATH_MIG:
+ case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
+ case MLX5_EVENT_TYPE_COMM_EST:
+ case MLX5_EVENT_TYPE_SQ_DRAINED:
+ case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
+ case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
+ return eqe->data.qp_srq.type;
+ case MLX5_EVENT_TYPE_CQ_ERROR:
+ return 0;
+ case MLX5_EVENT_TYPE_DCT_DRAINED:
+ return MLX5_EVENT_QUEUE_TYPE_DCT;
+ default:
+ return get_affiliated_event_obj_type(eqe);
+ }
+}
+
static u32 get_dec_obj_id(u64 obj_id)
{
return (obj_id & 0xffffffff);
@@ -1965,6 +2012,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
event_sub_arr[i]->xa_key_level1 = key_level1;
event_sub_arr[i]->xa_key_level2 = obj_id;
event_sub_arr[i]->is_obj_related = obj ? true : false;
+ INIT_LIST_HEAD(&event_sub_arr[i]->event_list);
}
/* Once all the allocations and the reservations for level 2
@@ -2175,10 +2223,174 @@ static int devx_umem_cleanup(struct ib_uobject *uobject,
return 0;
}
+static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
+ unsigned long event_type)
+{
+ __be64 *unaff_events;
+ int mask_entry;
+ int mask_bit;
+
+ if (!MLX5_CAP_GEN(dev, event_cap))
+ return is_legacy_unaffiliated_event_num(event_type);
+
+ unaff_events = MLX5_CAP_DEV_EVENT(dev,
+ user_unaffiliated_events);
+ WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
+
+ mask_entry = event_type / 64;
+ mask_bit = event_type % 64;
+
+ if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
+ return false;
+
+ return true;
+}
+
+static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
+{
+ struct mlx5_eqe *eqe = data;
+ u32 obj_id = 0;
+
+ switch (event_type) {
+ case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
+ case MLX5_EVENT_TYPE_PATH_MIG:
+ case MLX5_EVENT_TYPE_COMM_EST:
+ case MLX5_EVENT_TYPE_SQ_DRAINED:
+ case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+ obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
+ break;
+ case MLX5_EVENT_TYPE_DCT_DRAINED:
+ obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
+ break;
+ case MLX5_EVENT_TYPE_CQ_ERROR:
+ obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
+ break;
+ default:
+ obj_id = get_affiliated_event_obj_id(data);
+ break;
+ }
+
+ return obj_id;
+}
+
+static int deliver_event(struct devx_event_subscription *event_sub,
+ const void *data)
+{
+ struct ib_uobject *fd_uobj = event_sub->fd_uobj;
+ struct devx_async_event_file *ev_file;
+ struct devx_async_event_queue *ev_queue;
+ struct devx_async_event_data *event_data;
+ unsigned long flags;
+ bool omit_data;
+
+ ev_file = container_of(fd_uobj, struct devx_async_event_file,
+ uobj);
+ ev_queue = &ev_file->ev_queue;
+ omit_data = ev_queue->flags &
+ MLX5_IB_UAPI_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA;
+
+ if (omit_data) {
+ spin_lock_irqsave(&ev_queue->lock, flags);
+ if (!list_empty(&event_sub->event_list)) {
+ spin_unlock_irqrestore(&ev_queue->lock, flags);
+ return 0;
+ }
+
+ list_add_tail(&event_sub->event_list, &ev_queue->event_list);
+ spin_unlock_irqrestore(&ev_queue->lock, flags);
+ wake_up_interruptible(&ev_queue->poll_wait);
+ return 0;
+ }
+
+ event_data = kzalloc(sizeof(*event_data) +
+ (omit_data ? 0 : sizeof(struct mlx5_eqe)),
+ GFP_ATOMIC);
+ if (!event_data) {
+ spin_lock_irqsave(&ev_queue->lock, flags);
+ ev_queue->is_overflow_err = 1;
+ spin_unlock_irqrestore(&ev_queue->lock, flags);
+ return -ENOMEM;
+ }
+
+ event_data->hdr.cookie = event_sub->cookie;
+ memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
+
+ spin_lock_irqsave(&ev_queue->lock, flags);
+ list_add_tail(&event_data->list, &ev_queue->event_list);
+ spin_unlock_irqrestore(&ev_queue->lock, flags);
+ wake_up_interruptible(&ev_queue->poll_wait);
+
+ return 0;
+}
+
+static void dispatch_event_fd(struct list_head *fd_list,
+ const void *data)
+{
+ struct devx_event_subscription *item;
+
+ list_for_each_entry_rcu(item, fd_list, xa_list) {
+ if (!get_file_rcu((struct file *)item->fd_uobj->object))
+ continue;
+
+ if (item->eventfd) {
+ eventfd_signal(item->eventfd, 1);
+ fput(item->fd_uobj->object);
+ continue;
+ }
+
+ deliver_event(item, data);
+ fput(item->fd_uobj->object);
+ }
+}
+
static int devx_event_notifier(struct notifier_block *nb,
unsigned long event_type, void *data)
{
- return NOTIFY_DONE;
+ struct mlx5_devx_event_table *table;
+ struct mlx5_ib_dev *dev;
+ struct devx_event *event;
+ struct devx_obj_event *obj_event;
+ u16 obj_type = 0;
+ bool is_unaffiliated;
+ u32 obj_id;
+
+ /* Explicit filtering to kernel events which may occur frequently */
+ if (event_type == MLX5_EVENT_TYPE_CMD ||
+ event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
+ return NOTIFY_OK;
+
+ table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
+ dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
+ is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
+
+ if (!is_unaffiliated)
+ obj_type = get_event_obj_type(event_type, data);
+ event = xa_load(&table->event_xa, event_type | (obj_type << 16));
+ if (!event)
+ return NOTIFY_DONE;
+
+ if (is_unaffiliated) {
+ dispatch_event_fd(&event->unaffiliated_list, data);
+ return NOTIFY_OK;
+ }
+
+ obj_id = devx_get_obj_id_from_event(event_type, data);
+ rcu_read_lock();
+ obj_event = xa_load(&event->object_ids, obj_id);
+ if (!obj_event) {
+ rcu_read_unlock();
+ return NOTIFY_DONE;
+ }
+
+ dispatch_event_fd(&obj_event->obj_sub_list, data);
+
+ rcu_read_unlock();
+ return NOTIFY_OK;
}
void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
@@ -2306,17 +2518,153 @@ static const struct file_operations devx_async_cmd_event_fops = {
static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
size_t count, loff_t *pos)
{
- return -EINVAL;
+ struct devx_async_event_file *ev_file = filp->private_data;
+ struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
+ struct devx_event_subscription *event_sub;
+ struct devx_async_event_data *uninitialized_var(event);
+ int ret = 0;
+ size_t eventsz;
+ bool omit_data;
+ void *event_data;
+
+ omit_data = ev_queue->flags &
+ MLX5_IB_UAPI_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA;
+
+ spin_lock_irq(&ev_queue->lock);
+
+ if (ev_queue->is_overflow_err) {
+ ev_queue->is_overflow_err = 0;
+ spin_unlock_irq(&ev_queue->lock);
+ return -EOVERFLOW;
+ }
+
+ while (list_empty(&ev_queue->event_list)) {
+ spin_unlock_irq(&ev_queue->lock);
+
+ if (filp->f_flags & O_NONBLOCK)
+ return -EAGAIN;
+
+ if (wait_event_interruptible(ev_queue->poll_wait,
+ (!list_empty(&ev_queue->event_list) ||
+ ev_queue->is_destroyed))) {
+ return -ERESTARTSYS;
+ }
+
+ if (list_empty(&ev_queue->event_list) &&
+ ev_queue->is_destroyed)
+ return -EIO;
+
+ spin_lock_irq(&ev_queue->lock);
+ }
+
+ if (omit_data) {
+ event_sub = list_first_entry(&ev_queue->event_list,
+ struct devx_event_subscription,
+ event_list);
+ eventsz = sizeof(event_sub->cookie);
+ event_data = &event_sub->cookie;
+ } else {
+ event = list_first_entry(&ev_queue->event_list,
+ struct devx_async_event_data, list);
+ eventsz = sizeof(struct mlx5_eqe) +
+ sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
+ event_data = &event->hdr;
+ }
+
+ if (eventsz > count) {
+ spin_unlock_irq(&ev_queue->lock);
+ return -ENOSPC;
+ }
+
+ if (omit_data)
+ list_del_init(&event_sub->event_list);
+ else
+ list_del(&event->list);
+
+ spin_unlock_irq(&ev_queue->lock);
+
+ if (copy_to_user(buf, event_data, eventsz))
+ ret = -EFAULT;
+ else
+ ret = eventsz;
+
+ if (!omit_data)
+ kfree(event);
+ return ret;
}
static __poll_t devx_async_event_poll(struct file *filp,
struct poll_table_struct *wait)
{
- return 0;
+ struct devx_async_event_file *ev_file = filp->private_data;
+ struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
+ __poll_t pollflags = 0;
+
+ poll_wait(filp, &ev_queue->poll_wait, wait);
+
+ spin_lock_irq(&ev_queue->lock);
+ if (ev_queue->is_destroyed)
+ pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
+ else if (!list_empty(&ev_queue->event_list))
+ pollflags = EPOLLIN | EPOLLRDNORM;
+ spin_unlock_irq(&ev_queue->lock);
+
+ return pollflags;
}
static int devx_async_event_close(struct inode *inode, struct file *filp)
{
+ struct ib_uobject *uobj = filp->private_data;
+ struct devx_async_event_file *ev_file =
+ container_of(uobj, struct devx_async_event_file, uobj);
+ struct devx_event_subscription *event_sub, *event_sub_tmp;
+ struct devx_async_event_data *entry, *tmp;
+
+ mutex_lock(&ev_file->dev->devx_event_table.event_xa_lock);
+ /* delete the subscriptions which are related to this FD */
+ list_for_each_entry_safe(event_sub, event_sub_tmp,
+ &ev_file->subscribed_events_list, file_list) {
+ list_del_rcu(&event_sub->file_list);
+ list_del_rcu(&event_sub->xa_list);
+ if (event_sub->is_obj_related) {
+ struct devx_event *event;
+ struct devx_obj_event *xa_val_level2;
+
+ list_del_rcu(&event_sub->obj_list);
+
+ /* check whether the key level 1 for this obj_sub_list
+ * is empty
+ */
+ event = xa_load(
+ &ev_file->dev->devx_event_table.event_xa,
+ event_sub->xa_key_level1);
+ WARN_ON(!event);
+
+ xa_val_level2 = xa_load(&event->object_ids,
+ event_sub->xa_key_level2);
+ if (list_empty(&xa_val_level2->obj_sub_list)) {
+ xa_erase(&event->object_ids,
+ event_sub->xa_key_level2);
+ kfree_rcu(xa_val_level2, rcu);
+ }
+ }
+
+ if (event_sub->eventfd)
+ eventfd_ctx_put(event_sub->eventfd);
+
+ kfree_rcu(event_sub, rcu);
+ }
+ mutex_unlock(&ev_file->dev->devx_event_table.event_xa_lock);
+
+ /* free the pending events allocation */
+ if (!(ev_file->ev_queue.flags &
+ MLX5_IB_UAPI_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA)) {
+ spin_lock_irq(&ev_file->ev_queue.lock);
+ list_for_each_entry_safe(entry, tmp,
+ &ev_file->ev_queue.event_list, list)
+ kfree(entry); /* read can't come any nore */
+ spin_unlock_irq(&ev_file->ev_queue.lock);
+ }
uverbs_close_fd(filp);
return 0;
}
@@ -2351,6 +2699,17 @@ static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj,
static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj,
enum rdma_remove_reason why)
{
+ struct devx_async_event_file *ev_file =
+ container_of(uobj, struct devx_async_event_file,
+ uobj);
+ struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
+
+ spin_lock_irq(&ev_queue->lock);
+ ev_queue->is_destroyed = 1;
+ spin_unlock_irq(&ev_queue->lock);
+
+ if (why == RDMA_REMOVE_DRIVER_REMOVE)
+ wake_up_interruptible(&ev_queue->poll_wait);
return 0;
};
@@ -67,5 +67,10 @@ enum mlx5_ib_uapi_devx_create_event_channel_flags {
MLX5_IB_UAPI_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA = 1 << 0,
};
+struct mlx5_ib_uapi_devx_async_event_hdr {
+ __aligned_u64 cookie;
+ __u8 out_data[];
+};
+
#endif