From patchwork Thu Apr 25 12:58:53 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Heng Qi X-Patchwork-Id: 13643302 X-Patchwork-Delegate: kuba@kernel.org Received: from out30-111.freemail.mail.aliyun.com (out30-111.freemail.mail.aliyun.com [115.124.30.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id F0D8D149C64 for ; Thu, 25 Apr 2024 12:59:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=115.124.30.111 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1714049949; cv=none; b=aFNIZfoPr3Qbx3L6WtEmVMVe+bEDGx7G2qiRao3kOoZCWGbnBatVRSWw5tOdwwVEtjVl9T56ZR5I4CKYbIhlXiWuQg988QcjGy9iJOis6z1I6BYkbClENZ8MAItmDmZO0TfVlTy08wTU7/2r07+tftIy+vEnyWY3DOi0A3RYzK8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1714049949; c=relaxed/simple; bh=K84XKgK8KHbPWO0USuhlDpFa7QEvOPAwxpnWhm88NN8=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=Cw9jx/c/gSWfGtWT6gWnzsVKf0e/WNdYR8mLdsq6MB7jVd73B2R4D4mJ32dHEDmz8NDz5JYPpypYGvQECAj2Vd9vSmjTaX7GhfqSgoU3QKVx6jTGtXDjTbitwG6UNiPADyb60ydKZUeKhu7QddXR8W1qFT4B5Th4W8cnPHYEreg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=xcP3Pwkz; arc=none smtp.client-ip=115.124.30.111 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="xcP3Pwkz" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1714049938; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=bdjvXFYRnrSQReYwqmP/hdq3je0IOyW7L9cgHEAJbgA=; b=xcP3PwkzBhsKZn2HFkyyHbTZUUC+8xTwOXWcE6jpry4DKBT9JwLT7CIcVduDRYOM0/Z/qWy8JcIuJaAbleFJS3yEagWwtRDG39mwHgxRfl/JLjM2md0MZhFiu9nvgLJ6jTvsucDMk/usjJf54Zy/trZ3g/IXx6faN3Lc5kQ53Es= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R261e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=maildocker-contentspam033037067111;MF=hengqi@linux.alibaba.com;NM=1;PH=DS;RN=9;SR=0;TI=SMTPD_---0W5FdWV7_1714049936; Received: from localhost(mailfrom:hengqi@linux.alibaba.com fp:SMTPD_---0W5FdWV7_1714049936) by smtp.aliyun-inc.com; Thu, 25 Apr 2024 20:58:57 +0800 From: Heng Qi To: netdev@vger.kernel.org, virtualization@lists.linux.dev Cc: "Michael S . Tsirkin" , Jason Wang , Xuan Zhuo , Eric Dumazet , "David S . Miller" , Jakub Kicinski , Paolo Abeni Subject: [PATCH net-next 1/3] virtio_net: enable irq for the control vq Date: Thu, 25 Apr 2024 20:58:53 +0800 Message-Id: <20240425125855.87025-2-hengqi@linux.alibaba.com> X-Mailer: git-send-email 2.32.0.3.g01195cf9f In-Reply-To: <20240425125855.87025-1-hengqi@linux.alibaba.com> References: <20240425125855.87025-1-hengqi@linux.alibaba.com> Precedence: bulk X-Mailing-List: netdev@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: kuba@kernel.org Control vq polling request results consume more CPU. Especially when dim issues more control requests to the device, it's beneficial to the guest to enable control vq's irq. Suggested-by: Jason Wang Signed-off-by: Heng Qi --- drivers/net/virtio_net.c | 45 ++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index a4d3c76654a4..79a1b30c173c 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -287,6 +287,12 @@ struct virtnet_info { bool has_cvq; struct mutex cvq_lock; + /* Wait for the device to complete the request */ + struct completion completion; + + /* Work struct for acquisition of cvq processing results. */ + struct work_struct get_cvq; + /* Host can handle any s/g split between our header and packet data */ bool any_header_sg; @@ -520,6 +526,13 @@ static bool virtqueue_napi_complete(struct napi_struct *napi, return false; } +static void virtnet_cvq_done(struct virtqueue *cvq) +{ + struct virtnet_info *vi = cvq->vdev->priv; + + schedule_work(&vi->get_cvq); +} + static void skb_xmit_done(struct virtqueue *vq) { struct virtnet_info *vi = vq->vdev->priv; @@ -2036,6 +2049,20 @@ static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, return !oom; } +static void virtnet_get_cvq_work(struct work_struct *work) +{ + struct virtnet_info *vi = + container_of(work, struct virtnet_info, get_cvq); + unsigned int tmp; + void *res; + + mutex_lock(&vi->cvq_lock); + res = virtqueue_get_buf(vi->cvq, &tmp); + if (res) + complete(&vi->completion); + mutex_unlock(&vi->cvq_lock); +} + static void skb_recv_done(struct virtqueue *rvq) { struct virtnet_info *vi = rvq->vdev->priv; @@ -2531,7 +2558,7 @@ static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, struct scatterlist *out) { struct scatterlist *sgs[4], hdr, stat; - unsigned out_num = 0, tmp; + unsigned out_num = 0; int ret; /* Caller should know better */ @@ -2566,16 +2593,10 @@ static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, return vi->ctrl->status == VIRTIO_NET_OK; } - /* Spin for a response, the kick causes an ioport write, trapping - * into the hypervisor, so the request should be handled immediately. - */ - while (!virtqueue_get_buf(vi->cvq, &tmp) && - !virtqueue_is_broken(vi->cvq)) { - cond_resched(); - cpu_relax(); - } - mutex_unlock(&vi->cvq_lock); + + wait_for_completion(&vi->completion); + return vi->ctrl->status == VIRTIO_NET_OK; } @@ -4433,7 +4454,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi) /* Parameters for control virtqueue, if any */ if (vi->has_cvq) { - callbacks[total_vqs - 1] = NULL; + callbacks[total_vqs - 1] = virtnet_cvq_done; names[total_vqs - 1] = "control"; } @@ -4952,6 +4973,8 @@ static int virtnet_probe(struct virtio_device *vdev) if (vi->has_rss || vi->has_rss_hash_report) virtnet_init_default_rss(vi); + INIT_WORK(&vi->get_cvq, virtnet_get_cvq_work); + init_completion(&vi->completion); enable_rx_mode_work(vi); /* serialize netdev register + virtio_device_ready() with ndo_open() */ From patchwork Thu Apr 25 12:58:54 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Heng Qi X-Patchwork-Id: 13643301 X-Patchwork-Delegate: kuba@kernel.org Received: from out30-130.freemail.mail.aliyun.com (out30-130.freemail.mail.aliyun.com [115.124.30.130]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C068A1494DB for ; Thu, 25 Apr 2024 12:59:02 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=115.124.30.130 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1714049946; cv=none; b=S6+4QH+2bjZs7WFmUIzydKkX12YrEgaMJGuFpUlMFfqiLypqXGh00aWIBhM/mXVxnMFokGJov4lHs+zm/O21lbzaREFBvFl0/c61RS7+NCGRQTCqoy27kOa/DJMNQwu6tfnmed/6EdFdcZoKA3aEh1EUNL67DdWO1BW3oOwGiI8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1714049946; c=relaxed/simple; bh=M6SmxO5/6HWUBjl3LCU40SGHS4eWuCP5wqLJO/TeQ+c=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=a/iRV4tQckuWfkdsOvu+jDJGaRrbOu9r0J11tH6kRg5d+3kpAQWpZYR/shqTwO7yuLHkWnOjRS5Y9ffJlNljZoKVO+X5HcoalFOqqzaKCkmlI+x2NDNzeJ6t0y4WAsDT5kqzDFgZJx7UhzuMtxbFwjaGGfzUiNE6JnYGVG3BLTs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=BbbESW/R; arc=none smtp.client-ip=115.124.30.130 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="BbbESW/R" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1714049940; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=tJge7rhA7eFJ/ScRh9Cw+FRMa1b17ZVyKgD8pV+Xg+8=; b=BbbESW/RnNXvHJGslQCQGDX4KCMlzowjHkpzPrHc0MoVQ6lbvTSSwZL4rDU25tEymnPAXULmodPnTZaR6qBLR3tyib3x6dmFIer0xaj5Ce3X7Sxrs+heoKJaccXPrKT1wqI6a2A2n2eya7+zWflJGE8Bgl6HVriSQA8wEl0r2y0= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R191e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=maildocker-contentspam033037067109;MF=hengqi@linux.alibaba.com;NM=1;PH=DS;RN=9;SR=0;TI=SMTPD_---0W5FZNMk_1714049938; Received: from localhost(mailfrom:hengqi@linux.alibaba.com fp:SMTPD_---0W5FZNMk_1714049938) by smtp.aliyun-inc.com; Thu, 25 Apr 2024 20:58:59 +0800 From: Heng Qi To: netdev@vger.kernel.org, virtualization@lists.linux.dev Cc: "Michael S . Tsirkin" , Jason Wang , Xuan Zhuo , Eric Dumazet , "David S . Miller" , Jakub Kicinski , Paolo Abeni Subject: [PATCH net-next 2/3] virtio_net: fix possible dim status unrecoverable Date: Thu, 25 Apr 2024 20:58:54 +0800 Message-Id: <20240425125855.87025-3-hengqi@linux.alibaba.com> X-Mailer: git-send-email 2.32.0.3.g01195cf9f In-Reply-To: <20240425125855.87025-1-hengqi@linux.alibaba.com> References: <20240425125855.87025-1-hengqi@linux.alibaba.com> Precedence: bulk X-Mailing-List: netdev@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: kuba@kernel.org When the dim worker is scheduled, if it no longer needs to issue commands, dim may not be able to return to the working state later. For example, the following single queue scenario: 1. The dim worker of rxq0 is scheduled, and the dim status is changed to DIM_APPLY_NEW_PROFILE; 2. dim is disabled or parameters have not been modified; 3. virtnet_rx_dim_work exits directly; Then, even if net_dim is invoked again, it cannot work because the state is not restored to DIM_START_MEASURE. Fixes: 6208799553a8 ("virtio-net: support rx netdim") Signed-off-by: Heng Qi --- drivers/net/virtio_net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 79a1b30c173c..8f05bcf1d37d 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3650,9 +3650,9 @@ static void virtnet_rx_dim_work(struct work_struct *work) if (err) pr_debug("%s: Failed to send dim parameters on rxq%d\n", dev->name, qnum); - dim->state = DIM_START_MEASURE; } out: + dim->state = DIM_START_MEASURE; mutex_unlock(&rq->dim_lock); } From patchwork Thu Apr 25 12:58:55 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Heng Qi X-Patchwork-Id: 13643300 X-Patchwork-Delegate: kuba@kernel.org Received: from out30-97.freemail.mail.aliyun.com (out30-97.freemail.mail.aliyun.com [115.124.30.97]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 274AB149C4C for ; Thu, 25 Apr 2024 12:59:03 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=115.124.30.97 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1714049946; cv=none; b=l+27D0f/4baIunnCmrYp/LLKRTLqxIilgdqtZ88EmUcOkGxwLKCYtLki1uDph7SO3ivEn6gK0qGo9yn2UrVvrBNrEXg1R3M8xJZeACwawY2S91fFl8z6Tw9oYZi936OrZ34RhmlCkXYNBUm4XHbx4qNsfVUl2H/4FuXUe26sUhI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1714049946; c=relaxed/simple; bh=8BXeENDikJx2/yejgt+yDxMyJRSJMe9xVX73YK02RfA=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=AB8kes3YG4A0heNHISamEoibe2kKa6Map1h0wiyU8nWJFPbxNSeZLFScMFH8p0spVyUUn1U27BrlLVWLVPU/iVFg1+/mdr+3CYzwtOsMftDYRjkOgZCHyrZv6GvPlpLJz4e4xfxpIkVKcod9ozwnzL5K1tScoxXbx5XDQd+c++s= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=oSFiUltY; arc=none smtp.client-ip=115.124.30.97 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="oSFiUltY" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1714049941; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=1QNj3svvlNE4SI3QLwbE5brlaVr7j0KcHKFTkGq4w/I=; b=oSFiUltYME8kG2KwwsbET6rPU0w/vQcgQPsSuKUawKKq/wzEKfSISG3EOzsKkOMlhY/EfUnCFdah8xwmAY+rhzU2NTDl/y4cJXvgmgFGqwICjKOzci0cwgvy5Jr1LT9vzsNmQefiulKqlafUJtNas1+nKKKGe1T+hOyw1e+WQQc= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R181e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=maildocker-contentspam033037067110;MF=hengqi@linux.alibaba.com;NM=1;PH=DS;RN=9;SR=0;TI=SMTPD_---0W5FZmSN_1714049939; Received: from localhost(mailfrom:hengqi@linux.alibaba.com fp:SMTPD_---0W5FZmSN_1714049939) by smtp.aliyun-inc.com; Thu, 25 Apr 2024 20:59:00 +0800 From: Heng Qi To: netdev@vger.kernel.org, virtualization@lists.linux.dev Cc: "Michael S . Tsirkin" , Jason Wang , Xuan Zhuo , Eric Dumazet , "David S . Miller" , Jakub Kicinski , Paolo Abeni Subject: [PATCH net-next 3/3] virtio_net: improve dim command request efficiency Date: Thu, 25 Apr 2024 20:58:55 +0800 Message-Id: <20240425125855.87025-4-hengqi@linux.alibaba.com> X-Mailer: git-send-email 2.32.0.3.g01195cf9f In-Reply-To: <20240425125855.87025-1-hengqi@linux.alibaba.com> References: <20240425125855.87025-1-hengqi@linux.alibaba.com> Precedence: bulk X-Mailing-List: netdev@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: kuba@kernel.org Currently, ctrlq processes commands in a synchronous manner, which increases the delay of dim commands when configuring multi-queue VMs, which in turn causes the CPU utilization to increase and interferes with the performance of dim. Therefore we asynchronously process ctlq's dim commands. Signed-off-by: Heng Qi --- drivers/net/virtio_net.c | 229 +++++++++++++++++++++++++++++++++++---- 1 file changed, 209 insertions(+), 20 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 8f05bcf1d37d..6cd72c0119e6 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -138,6 +138,14 @@ struct virtnet_interrupt_coalesce { u32 max_usecs; }; +struct virtnet_coal_node { + struct virtio_net_ctrl_hdr hdr; + virtio_net_ctrl_ack status; + struct virtio_net_ctrl_coal_vq coal_vqs; + bool is_wait; + struct list_head list; +}; + /* The dma information of pages allocated at a time. */ struct virtnet_rq_dma { dma_addr_t addr; @@ -337,6 +345,14 @@ struct virtnet_info { struct virtnet_interrupt_coalesce intr_coal_tx; struct virtnet_interrupt_coalesce intr_coal_rx; + /* Free nodes used for concurrent delivery */ + struct mutex coal_free_lock; + struct list_head coal_free_list; + + /* Filled when there are no free nodes or cvq buffers */ + struct mutex coal_wait_lock; + struct list_head coal_wait_list; + unsigned long guest_offloads; unsigned long guest_offloads_capable; @@ -2049,17 +2065,108 @@ static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, return !oom; } +static void __virtnet_add_dim_command(struct virtnet_info *vi, + struct virtnet_coal_node *ctrl) +{ + struct scatterlist *sgs[4], hdr, stat, out; + unsigned int out_num = 0; + int ret; + + BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); + + ctrl->hdr.class = VIRTIO_NET_CTRL_NOTF_COAL; + ctrl->hdr.cmd = VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET; + + sg_init_one(&hdr, &ctrl->hdr, sizeof(ctrl->hdr)); + sgs[out_num++] = &hdr; + + sg_init_one(&out, &ctrl->coal_vqs, sizeof(ctrl->coal_vqs)); + sgs[out_num++] = &out; + + ctrl->status = VIRTIO_NET_OK; + sg_init_one(&stat, &ctrl->status, sizeof(ctrl->status)); + sgs[out_num] = &stat; + + BUG_ON(out_num + 1 > ARRAY_SIZE(sgs)); + ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, ctrl, GFP_ATOMIC); + if (ret < 0) { + dev_warn(&vi->vdev->dev, "Failed to add sgs for command vq: %d\n.", ret); + return; + } + + virtqueue_kick(vi->cvq); +} + +static void virtnet_add_dim_command(struct virtnet_info *vi, + struct virtnet_coal_node *ctrl) +{ + mutex_lock(&vi->cvq_lock); + __virtnet_add_dim_command(vi, ctrl); + mutex_unlock(&vi->cvq_lock); +} + +static void virtnet_process_dim_cmd(struct virtnet_info *vi, void *res) +{ + struct virtnet_coal_node *node; + u16 qnum; + + node = (struct virtnet_coal_node *)res; + qnum = le16_to_cpu(node->coal_vqs.vqn) / 2; + + mutex_lock(&vi->rq[qnum].dim_lock); + vi->rq[qnum].intr_coal.max_usecs = + le32_to_cpu(node->coal_vqs.coal.max_usecs); + vi->rq[qnum].intr_coal.max_packets = + le32_to_cpu(node->coal_vqs.coal.max_packets); + vi->rq[qnum].dim.state = DIM_START_MEASURE; + mutex_unlock(&vi->rq[qnum].dim_lock); + + if (!node->is_wait) { + mutex_lock(&vi->coal_free_lock); + list_add(&node->list, &vi->coal_free_list); + mutex_unlock(&vi->coal_free_lock); + } else { + kfree(node); + } +} + static void virtnet_get_cvq_work(struct work_struct *work) { struct virtnet_info *vi = container_of(work, struct virtnet_info, get_cvq); + struct virtnet_coal_node *wait_coal; + bool valid = false; unsigned int tmp; void *res; mutex_lock(&vi->cvq_lock); - res = virtqueue_get_buf(vi->cvq, &tmp); - if (res) - complete(&vi->completion); + while ((res = virtqueue_get_buf(vi->cvq, &tmp)) != NULL) { + if (res == ((void *)vi)) + complete(&vi->completion); + else + virtnet_process_dim_cmd(vi, res); + + valid = true; + } + + if (!valid) { + mutex_unlock(&vi->cvq_lock); + return; + } + + mutex_lock(&vi->coal_wait_lock); + while (!list_empty(&vi->coal_wait_list)) { + if (vi->cvq->num_free < 3) + goto out_get_cvq; + + wait_coal = list_first_entry(&vi->coal_wait_list, + struct virtnet_coal_node, list); + list_del(&wait_coal->list); + __virtnet_add_dim_command(vi, wait_coal); + } + +out_get_cvq: + mutex_unlock(&vi->coal_wait_lock); mutex_unlock(&vi->cvq_lock); } @@ -3625,35 +3732,73 @@ static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi, return 0; } +static void virtnet_put_wait_coal(struct virtnet_info *vi, + struct receive_queue *rq, + struct dim_cq_moder moder) +{ + struct virtnet_coal_node *wait_node; + + wait_node = kzalloc(sizeof(*wait_node), GFP_KERNEL); + if (!wait_node) { + rq->dim.state = DIM_START_MEASURE; + return; + } + + wait_node->is_wait = true; + wait_node->coal_vqs.vqn = cpu_to_le16(rxq2vq(rq - vi->rq)); + wait_node->coal_vqs.coal.max_usecs = cpu_to_le32(moder.usec); + wait_node->coal_vqs.coal.max_packets = cpu_to_le32(moder.pkts); + mutex_lock(&vi->coal_wait_lock); + list_add_tail(&wait_node->list, &vi->coal_wait_list); + mutex_unlock(&vi->coal_wait_lock); +} + static void virtnet_rx_dim_work(struct work_struct *work) { struct dim *dim = container_of(work, struct dim, work); struct receive_queue *rq = container_of(dim, struct receive_queue, dim); struct virtnet_info *vi = rq->vq->vdev->priv; - struct net_device *dev = vi->dev; + struct virtnet_coal_node *avail_coal; struct dim_cq_moder update_moder; - int qnum, err; - qnum = rq - vi->rq; + update_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); mutex_lock(&rq->dim_lock); - if (!rq->dim_enabled) - goto out; - - update_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); - if (update_moder.usec != rq->intr_coal.max_usecs || - update_moder.pkts != rq->intr_coal.max_packets) { - err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, qnum, - update_moder.usec, - update_moder.pkts); - if (err) - pr_debug("%s: Failed to send dim parameters on rxq%d\n", - dev->name, qnum); + if (!rq->dim_enabled || + (update_moder.usec == rq->intr_coal.max_usecs && + update_moder.pkts == rq->intr_coal.max_packets)) { + rq->dim.state = DIM_START_MEASURE; + mutex_unlock(&rq->dim_lock); + return; } -out: - dim->state = DIM_START_MEASURE; mutex_unlock(&rq->dim_lock); + + mutex_lock(&vi->cvq_lock); + if (vi->cvq->num_free < 3) { + virtnet_put_wait_coal(vi, rq, update_moder); + mutex_unlock(&vi->cvq_lock); + return; + } + mutex_unlock(&vi->cvq_lock); + + mutex_lock(&vi->coal_free_lock); + if (list_empty(&vi->coal_free_list)) { + virtnet_put_wait_coal(vi, rq, update_moder); + mutex_unlock(&vi->coal_free_lock); + return; + } + + avail_coal = list_first_entry(&vi->coal_free_list, + struct virtnet_coal_node, list); + avail_coal->coal_vqs.vqn = cpu_to_le16(rxq2vq(rq - vi->rq)); + avail_coal->coal_vqs.coal.max_usecs = cpu_to_le32(update_moder.usec); + avail_coal->coal_vqs.coal.max_packets = cpu_to_le32(update_moder.pkts); + + list_del(&avail_coal->list); + mutex_unlock(&vi->coal_free_lock); + + virtnet_add_dim_command(vi, avail_coal); } static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) @@ -4748,6 +4893,45 @@ static const struct xdp_metadata_ops virtnet_xdp_metadata_ops = { .xmo_rx_hash = virtnet_xdp_rx_hash, }; +static void virtnet_del_coal_free_list(struct virtnet_info *vi) +{ + struct virtnet_coal_node *coal_node, *tmp; + + list_for_each_entry_safe(coal_node, tmp, &vi->coal_free_list, list) { + list_del(&coal_node->list); + kfree(coal_node); + } +} + +static int virtnet_init_coal_list(struct virtnet_info *vi) +{ + struct virtnet_coal_node *coal_node; + int batch_dim_nums; + int i; + + INIT_LIST_HEAD(&vi->coal_free_list); + mutex_init(&vi->coal_free_lock); + + INIT_LIST_HEAD(&vi->coal_wait_list); + mutex_init(&vi->coal_wait_lock); + + if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) + return 0; + + batch_dim_nums = min((unsigned int)vi->max_queue_pairs, + virtqueue_get_vring_size(vi->cvq) / 3); + for (i = 0; i < batch_dim_nums; i++) { + coal_node = kzalloc(sizeof(*coal_node), GFP_KERNEL); + if (!coal_node) { + virtnet_del_coal_free_list(vi); + return -ENOMEM; + } + list_add(&coal_node->list, &vi->coal_free_list); + } + + return 0; +} + static int virtnet_probe(struct virtio_device *vdev) { int i, err = -ENOMEM; @@ -4932,6 +5116,9 @@ static int virtnet_probe(struct virtio_device *vdev) if (err) goto free; + if (virtnet_init_coal_list(vi)) + goto free; + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { vi->intr_coal_rx.max_usecs = 0; vi->intr_coal_tx.max_usecs = 0; @@ -5081,6 +5268,8 @@ static void virtnet_remove(struct virtio_device *vdev) net_failover_destroy(vi->failover); + virtnet_del_coal_free_list(vi); + remove_vq_common(vi); free_netdev(vi->dev);