From patchwork Tue Nov 30 12:46:30 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 12647229 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id ADA7BC433F5 for ; Tue, 30 Nov 2021 12:47:23 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S241876AbhK3Mul (ORCPT ); Tue, 30 Nov 2021 07:50:41 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59672 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231269AbhK3MuR (ORCPT ); Tue, 30 Nov 2021 07:50:17 -0500 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 972A4C061574 for ; Tue, 30 Nov 2021 04:46:46 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=i6BSDgSOKIl95ryb5Puqf0SnBy2urheC/nx+MGMFmxc=; b=hHEaAdTjDEkBSbSgQOB3tePXMa xyQ7DXHLvDyJ4ntPR4+bKakCCzzycVHH69ZzijE2vC58OLCGFCOyGCeXcDCZrjN63Yk89gGRILuPQ t6h8JZSqfJtPrzwoBmfN32rXF6ahay/coPKAqhQ4zcqpvuZKMZ7nDFfy86gswJ3S7i6brSCOPFL+8 eCY67N+ZseMBCqQAG4yYcehb/7BvplzpDy0rFKz3rM9Kf6KSkC8+HY+t3PvY7bSOSI7Ccxol/rPac pyois1pKcP9SYykktI9z2K1qXtRU1bxJxmxJ9SHtMkzjjPOZPpq5+1f0fytwO9Rmrjtfw6Gv5A/YO rPtCrPQQ==; Received: from [2001:4bb8:184:4a23:f08:7851:5d49:c683] (helo=localhost) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1ms2Wx-00CFan-9j; Tue, 30 Nov 2021 12:46:44 +0000 From: Christoph Hellwig To: Jens Axboe Cc: Paolo Valente , Jan Kara , linux-block@vger.kernel.org Subject: [PATCH 1/7] block: remove the nr_task field from struct io_context Date: Tue, 30 Nov 2021 13:46:30 +0100 Message-Id: <20211130124636.2505904-2-hch@lst.de> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211130124636.2505904-1-hch@lst.de> References: <20211130124636.2505904-1-hch@lst.de> MIME-Version: 1.0 X-SRS-Rewrite: SMTP reverse-path rewritten from by casper.infradead.org. See http://www.infradead.org/rpr.html Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Nothing ever looks at ->nr_tasks, so remove it. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara --- block/blk-ioc.c | 3 --- include/linux/iocontext.h | 1 - 2 files changed, 4 deletions(-) diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 536fb496ad763..96336c2134efa 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -207,7 +207,6 @@ void exit_io_context(struct task_struct *task) task->io_context = NULL; task_unlock(task); - atomic_dec(&ioc->nr_tasks); put_io_context_active(ioc); } @@ -259,7 +258,6 @@ static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) return NULL; atomic_long_set(&ioc->refcount, 1); - atomic_set(&ioc->nr_tasks, 1); atomic_set(&ioc->active_ref, 1); spin_lock_init(&ioc->lock); INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC); @@ -339,7 +337,6 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) if (clone_flags & CLONE_IO) { atomic_long_inc(&ioc->refcount); atomic_inc(&ioc->active_ref); - atomic_inc(&ioc->nr_tasks); tsk->io_context = ioc; } else if (ioprio_valid(ioc->ioprio)) { tsk->io_context = alloc_io_context(GFP_KERNEL, NUMA_NO_NODE); diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index c1229fbd6691c..82c7f4f5f4f59 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -99,7 +99,6 @@ struct io_cq { struct io_context { atomic_long_t refcount; atomic_t active_ref; - atomic_t nr_tasks; /* all the fields below are protected by this lock */ spinlock_t lock; From patchwork Tue Nov 30 12:46:31 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 12647237 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id DBD82C43219 for ; Tue, 30 Nov 2021 12:47:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230246AbhK3Mum (ORCPT ); Tue, 30 Nov 2021 07:50:42 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59680 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231280AbhK3MuT (ORCPT ); Tue, 30 Nov 2021 07:50:19 -0500 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 8E4D6C061748 for ; Tue, 30 Nov 2021 04:46:47 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=NbgYne6q4541wB0r0paU4D08lIBCJKTKuAZf9zu2Hi0=; b=YgCyNzY9D//mbdgnWSGRrEuQUm l8rCMEeU8W8Kuha6QqDWXCxOyYD3SZj9xdZTzjFC6JK9f3/5Xc3qAOT3etGyl47W1XN1hmGeHFGkc uOTMgmuBfLFJxPTruMA90Q3ZB19j5bIf6MbwT9AEnfqZjH4ak8YZ9Mm/ZOAKwSovWnty7lhxwbxWL FaiXtusj0721ZqvPYib7veg4lSUVHcqZSDK/luWN1QOwgCHZxUeB+hhELQ0PXx4hnRVhLharYkqWP AQ1+EfbCTLQaF7vkYPnVp0WoV9cnF3xxNm04nopqsVv4FMRzemHn7uIwZXMQGr5ITF53JT/PXwXdj 3kXGu0SA==; Received: from [2001:4bb8:184:4a23:f08:7851:5d49:c683] (helo=localhost) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1ms2Wy-00CFb7-GU; Tue, 30 Nov 2021 12:46:45 +0000 From: Christoph Hellwig To: Jens Axboe Cc: Paolo Valente , Jan Kara , linux-block@vger.kernel.org Subject: [PATCH 2/7] block: simplify struct io_context refcounting Date: Tue, 30 Nov 2021 13:46:31 +0100 Message-Id: <20211130124636.2505904-3-hch@lst.de> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211130124636.2505904-1-hch@lst.de> References: <20211130124636.2505904-1-hch@lst.de> MIME-Version: 1.0 X-SRS-Rewrite: SMTP reverse-path rewritten from by casper.infradead.org. See http://www.infradead.org/rpr.html Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Don't hold a reference to ->refcount for each active reference, but just one for all active references. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara --- block/blk-ioc.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 96336c2134efa..9cde3906be3c6 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -180,10 +180,8 @@ static void put_io_context_active(struct io_context *ioc) { struct io_cq *icq; - if (!atomic_dec_and_test(&ioc->active_ref)) { - put_io_context(ioc); + if (!atomic_dec_and_test(&ioc->active_ref)) return; - } spin_lock_irq(&ioc->lock); hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) { @@ -335,7 +333,6 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) * Share io context with parent, if CLONE_IO is set */ if (clone_flags & CLONE_IO) { - atomic_long_inc(&ioc->refcount); atomic_inc(&ioc->active_ref); tsk->io_context = ioc; } else if (ioprio_valid(ioc->ioprio)) { From patchwork Tue Nov 30 12:46:32 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 12647231 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 967DCC4332F for ; Tue, 30 Nov 2021 12:47:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231181AbhK3Mum (ORCPT ); Tue, 30 Nov 2021 07:50:42 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59692 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231459AbhK3MuT (ORCPT ); Tue, 30 Nov 2021 07:50:19 -0500 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D6D05C06174A for ; Tue, 30 Nov 2021 04:46:49 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=U10Z8+JHPFXXgLEhho7zQZjwErerxfS6SmCDmVZWe3k=; b=JZUsWnie9IPLz40CzqAizcr/U8 ZhPjMlCKR8C+5lhFzwX+ba48LzhK2+bc67W4X39ihBz0ITsZ/O/4rlONBrSbmLCjRuZheIHPdQ7mA uMaNFjbfzBeQTHv5V7b9MYT2zho+pw7/sXE3TxaZ11jWeNXooMDdRKaXHvQqLCTjDcywAQjknvhZu tLgRBS2KhV5TDHZAsCFhQPQtoDni4113uMmK4GRWVi5sW3ECL7JRyIGMP8HMhgESkO7VjPBiKQpQ0 iKOFm1y1zJrpMc4RotBE6XROTCQcUJsmY2QG7fB4JDkFJ97AtWnAuQ1WTyWHW5bglA9o44Xs+TrjO mWmvQNSg==; Received: from [2001:4bb8:184:4a23:f08:7851:5d49:c683] (helo=localhost) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1ms2X0-00CFbm-PQ; Tue, 30 Nov 2021 12:46:47 +0000 From: Christoph Hellwig To: Jens Axboe Cc: Paolo Valente , Jan Kara , linux-block@vger.kernel.org Subject: [PATCH 3/7] block: refactor put_iocontext_active Date: Tue, 30 Nov 2021 13:46:32 +0100 Message-Id: <20211130124636.2505904-4-hch@lst.de> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211130124636.2505904-1-hch@lst.de> References: <20211130124636.2505904-1-hch@lst.de> MIME-Version: 1.0 X-SRS-Rewrite: SMTP reverse-path rewritten from by casper.infradead.org. See http://www.infradead.org/rpr.html Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Factor out a ioc_exit_icqs helper to tear down the icqs and the fold the rest of put_iocontext_active into exit_io_context. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara --- block/blk-ioc.c | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 9cde3906be3c6..0380e33930e31 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -54,6 +54,16 @@ static void ioc_exit_icq(struct io_cq *icq) icq->flags |= ICQ_EXITED; } +static void ioc_exit_icqs(struct io_context *ioc) +{ + struct io_cq *icq; + + spin_lock_irq(&ioc->lock); + hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) + ioc_exit_icq(icq); + spin_unlock_irq(&ioc->lock); +} + /* * Release an icq. Called with ioc locked for blk-mq, and with both ioc * and queue locked for legacy. @@ -169,32 +179,6 @@ void put_io_context(struct io_context *ioc) } EXPORT_SYMBOL_GPL(put_io_context); -/** - * put_io_context_active - put active reference on ioc - * @ioc: ioc of interest - * - * Put an active reference to an ioc. If active reference reaches zero after - * put, @ioc can never issue further IOs and ioscheds are notified. - */ -static void put_io_context_active(struct io_context *ioc) -{ - struct io_cq *icq; - - if (!atomic_dec_and_test(&ioc->active_ref)) - return; - - spin_lock_irq(&ioc->lock); - hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) { - if (icq->flags & ICQ_EXITED) - continue; - - ioc_exit_icq(icq); - } - spin_unlock_irq(&ioc->lock); - - put_io_context(ioc); -} - /* Called by the exiting task */ void exit_io_context(struct task_struct *task) { @@ -205,7 +189,10 @@ void exit_io_context(struct task_struct *task) task->io_context = NULL; task_unlock(task); - put_io_context_active(ioc); + if (atomic_dec_and_test(&ioc->active_ref)) { + ioc_exit_icqs(ioc); + put_io_context(ioc); + } } static void __ioc_clear_queue(struct list_head *icq_list) From patchwork Tue Nov 30 12:46:33 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 12647235 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 213DAC43217 for ; Tue, 30 Nov 2021 12:47:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229463AbhK3Mum (ORCPT ); Tue, 30 Nov 2021 07:50:42 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59694 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231591AbhK3MuT (ORCPT ); Tue, 30 Nov 2021 07:50:19 -0500 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 121EBC061756 for ; Tue, 30 Nov 2021 04:46:51 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=4dOr9mVd/x4gQxXkD4EqAd0AYwdkpTESj/mQN52XBZs=; b=GUD04Fg2/+p9JC9MwbWEBsmPRD gKG2ihM+rVVfxK9fBJPTZVowcJPIDsFktg+7s2YHV88q59ISFskrIQRCjZksDnK7ZxM3RiQ8o8ej+ iqnGSxLHw/cVdNOWwQjUkucrG+sO4uxxtrmeUsAPdDKS1BJvKgn+WFbnQb/uozlZg4QNJytlq5LW8 +3kBC/fREshKfyrUczZNpMD08jLnwfkcAjvvgBab/dd5zC4n4dePRKo5ace+S41wjT98N0z4bDTKQ q9FaYU7AP+jrXboL8ygMXboP0mpdl1SGwc+VfzBwv9A5JAGnDpSV339HeHb6WTQUePwEfRK27nAyZ 7dPe+cUw==; Received: from [2001:4bb8:184:4a23:f08:7851:5d49:c683] (helo=localhost) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1ms2X1-00CFc5-V0; Tue, 30 Nov 2021 12:46:48 +0000 From: Christoph Hellwig To: Jens Axboe Cc: Paolo Valente , Jan Kara , linux-block@vger.kernel.org Subject: [PATCH 4/7] block: remove the NULL ioc check in put_io_context Date: Tue, 30 Nov 2021 13:46:33 +0100 Message-Id: <20211130124636.2505904-5-hch@lst.de> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211130124636.2505904-1-hch@lst.de> References: <20211130124636.2505904-1-hch@lst.de> MIME-Version: 1.0 X-SRS-Rewrite: SMTP reverse-path rewritten from by casper.infradead.org. See http://www.infradead.org/rpr.html Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org No caller passes in a NULL pointer, so remove the check. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara --- block/blk-ioc.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 0380e33930e31..04f3d2b0ca7db 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -155,9 +155,6 @@ void put_io_context(struct io_context *ioc) unsigned long flags; bool free_ioc = false; - if (ioc == NULL) - return; - BUG_ON(atomic_long_read(&ioc->refcount) <= 0); /* From patchwork Tue Nov 30 12:46:34 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 12647233 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7E73CC4167B for ; Tue, 30 Nov 2021 12:47:25 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231256AbhK3Mun (ORCPT ); Tue, 30 Nov 2021 07:50:43 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59732 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232089AbhK3MuY (ORCPT ); Tue, 30 Nov 2021 07:50:24 -0500 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 43B9DC061757 for ; Tue, 30 Nov 2021 04:46:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=9vrej7Co0BS/WRJ/OE2xz0kPr8K/dbE1m0sG5liWbX8=; b=CvKzYQ2qTiQQ8ffKeEpBPsJKhb 8SVQ3eP2lK+HdlvtKcLQG4IlHXV7Wc59ulfjaUHUg/mDv2Z0zwiG6qHLaJLQFw2mFbx1BbltCUF84 oFXlPMSUJKasvG5zY0GA7Fin4PhvQ6n6rlEGEA5m15CaB6hnEI4WiNrI+D7AzmHRrFZibRHEuN7EC jjaD5Q/a2mcMUkaxfiiVot1dBO1cPLn/QUkH5NUPo/rOs0K0QxUL129zsUuLkMSUQtWbwXZ239lUW TXkJqKJCnuXRUXW/+if7uvvemdnsHxCYIeiPjc5FC81WnIlvJu/pieytJwfb22UUWwi2X89+eo/Ox DNaqENQw==; Received: from [2001:4bb8:184:4a23:f08:7851:5d49:c683] (helo=localhost) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1ms2X3-00CFcR-BL; Tue, 30 Nov 2021 12:46:50 +0000 From: Christoph Hellwig To: Jens Axboe Cc: Paolo Valente , Jan Kara , linux-block@vger.kernel.org Subject: [PATCH 5/7] block: refactor put_io_context Date: Tue, 30 Nov 2021 13:46:34 +0100 Message-Id: <20211130124636.2505904-6-hch@lst.de> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211130124636.2505904-1-hch@lst.de> References: <20211130124636.2505904-1-hch@lst.de> MIME-Version: 1.0 X-SRS-Rewrite: SMTP reverse-path rewritten from by casper.infradead.org. See http://www.infradead.org/rpr.html Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Move the code to delay freeing the icqs into a separate helper. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara --- block/blk-ioc.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 04f3d2b0ca7db..902bca2b273ba 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -143,6 +143,24 @@ static void ioc_release_fn(struct work_struct *work) kmem_cache_free(iocontext_cachep, ioc); } +/* + * Releasing icqs requires reverse order double locking and we may already be + ( holding a queue_lock. Do it asynchronously from a workqueue. + */ +static bool ioc_delay_free(struct io_context *ioc) +{ + unsigned long flags; + + spin_lock_irqsave(&ioc->lock, flags); + if (!hlist_empty(&ioc->icq_list)) { + queue_work(system_power_efficient_wq, &ioc->release_work); + spin_unlock_irqrestore(&ioc->lock, flags); + return true; + } + spin_unlock_irqrestore(&ioc->lock, flags); + return false; +} + /** * put_io_context - put a reference of io_context * @ioc: io_context to put @@ -152,26 +170,8 @@ static void ioc_release_fn(struct work_struct *work) */ void put_io_context(struct io_context *ioc) { - unsigned long flags; - bool free_ioc = false; - BUG_ON(atomic_long_read(&ioc->refcount) <= 0); - - /* - * Releasing ioc requires reverse order double locking and we may - * already be holding a queue_lock. Do it asynchronously from wq. - */ - if (atomic_long_dec_and_test(&ioc->refcount)) { - spin_lock_irqsave(&ioc->lock, flags); - if (!hlist_empty(&ioc->icq_list)) - queue_work(system_power_efficient_wq, - &ioc->release_work); - else - free_ioc = true; - spin_unlock_irqrestore(&ioc->lock, flags); - } - - if (free_ioc) + if (atomic_long_dec_and_test(&ioc->refcount) && !ioc_delay_free(ioc)) kmem_cache_free(iocontext_cachep, ioc); } EXPORT_SYMBOL_GPL(put_io_context); From patchwork Tue Nov 30 12:46:35 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 12647241 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A5782C4167E for ; Tue, 30 Nov 2021 12:47:25 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231269AbhK3Mun (ORCPT ); Tue, 30 Nov 2021 07:50:43 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59734 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232057AbhK3MuY (ORCPT ); Tue, 30 Nov 2021 07:50:24 -0500 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 937ECC061758 for ; Tue, 30 Nov 2021 04:46:53 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=iL5jVPLBFjqePJhzbCCqDcgjcEvuK8+6S3TmGqGptZw=; b=JHwZEpDugvXi8QTLNknIS+Jkar /Hvg13eiTERV65ru9e25M1MoMEnHd1Yf/VWzdqKs81DmC3isgGQxW7QjQrKbxbB5wbYIX7wZBwOhu EMlC/SOHolJVQ0pqna0rQsF+XtjRjyXQvBzFvGQEn6mMsSiik+e8GtqO0+9KVTYRlm6VyAIgdA+c0 ovFiZlWz6aHFWWyBKgze6a+RbVFVQ0EM7+iXHConypDtcUw2SnG2UD8OaHsEDn1IytavctTx9hbbR wM07JR+zmqslzNd0rc49TGxhs604++nYgZDP0NitRxAAX8k0vyRioKQZP6PksnihX3GqKBJIgUWlT rNBjW38A==; Received: from [2001:4bb8:184:4a23:f08:7851:5d49:c683] (helo=localhost) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1ms2X4-00CFcq-K2; Tue, 30 Nov 2021 12:46:51 +0000 From: Christoph Hellwig To: Jens Axboe Cc: Paolo Valente , Jan Kara , linux-block@vger.kernel.org Subject: [PATCH 6/7] block: cleanup ioc_clear_queue Date: Tue, 30 Nov 2021 13:46:35 +0100 Message-Id: <20211130124636.2505904-7-hch@lst.de> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211130124636.2505904-1-hch@lst.de> References: <20211130124636.2505904-1-hch@lst.de> MIME-Version: 1.0 X-SRS-Rewrite: SMTP reverse-path rewritten from by casper.infradead.org. See http://www.infradead.org/rpr.html Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Fold __ioc_clear_queue into ioc_clear_queue, remove the spurious RCU criticial section and unify the irq locking style. Signed-off-by: Christoph Hellwig --- block/blk-ioc.c | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 902bca2b273ba..32ae006e1b3e8 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -192,27 +192,6 @@ void exit_io_context(struct task_struct *task) } } -static void __ioc_clear_queue(struct list_head *icq_list) -{ - unsigned long flags; - - rcu_read_lock(); - while (!list_empty(icq_list)) { - struct io_cq *icq = list_entry(icq_list->next, - struct io_cq, q_node); - struct io_context *ioc = icq->ioc; - - spin_lock_irqsave(&ioc->lock, flags); - if (icq->flags & ICQ_DESTROYED) { - spin_unlock_irqrestore(&ioc->lock, flags); - continue; - } - ioc_destroy_icq(icq); - spin_unlock_irqrestore(&ioc->lock, flags); - } - rcu_read_unlock(); -} - /** * ioc_clear_queue - break any ioc association with the specified queue * @q: request_queue being cleared @@ -227,7 +206,15 @@ void ioc_clear_queue(struct request_queue *q) list_splice_init(&q->icq_list, &icq_list); spin_unlock_irq(&q->queue_lock); - __ioc_clear_queue(&icq_list); + while (!list_empty(&icq_list)) { + struct io_cq *icq = + list_entry(icq_list.next, struct io_cq, q_node); + + spin_lock_irq(&icq->ioc->lock); + if (!(icq->flags & ICQ_DESTROYED)) + ioc_destroy_icq(icq); + spin_unlock_irq(&icq->ioc->lock); + } } static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) From patchwork Tue Nov 30 12:46:36 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 12647239 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8FF54C4321E for ; Tue, 30 Nov 2021 12:47:25 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231253AbhK3Mun (ORCPT ); Tue, 30 Nov 2021 07:50:43 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59736 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232424AbhK3MuZ (ORCPT ); Tue, 30 Nov 2021 07:50:25 -0500 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id CEDEDC061759 for ; Tue, 30 Nov 2021 04:46:54 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=6awz+KTH6K8Gbow65xcfkEOlvwbnrYcVy3c8n+wdDkM=; b=Iut4JgOFqdqaZSWXcFXoyNibt7 mUfOtNUS/GfSzx0yF4U+Gk/VeGRtQiRd6UcMDynzrra2fyWlQ/WzKyR4OhNcnAU+PseZcMwu0BpeV 3SMKJaeuZKY2kKm1U1+ogySjrxYEAtIW8iHZE7SIOovLsFNKEnNgMpamPWSDqY2NVV/tvnK6rh/BN v+vzNSsbfdyswmEVd5mdYbJs5g7/jaKmA9xE/Af3Vkf/e2FqSr97X8Eez6HIHyc73F2rRf1tg+Jbe /fHCS0B8Rf7kBVFmRLwjjteKfUZgteVxDYV9EgUWXINLzPtXzeJHq+8egaLNJhp+aSJSLi1skJjCC cVgj+ZLg==; Received: from [2001:4bb8:184:4a23:f08:7851:5d49:c683] (helo=localhost) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1ms2X5-00CFdf-Rm; Tue, 30 Nov 2021 12:46:52 +0000 From: Christoph Hellwig To: Jens Axboe Cc: Paolo Valente , Jan Kara , linux-block@vger.kernel.org Subject: [PATCH 7/7] block: only build the icq tracking code when needed Date: Tue, 30 Nov 2021 13:46:36 +0100 Message-Id: <20211130124636.2505904-8-hch@lst.de> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211130124636.2505904-1-hch@lst.de> References: <20211130124636.2505904-1-hch@lst.de> MIME-Version: 1.0 X-SRS-Rewrite: SMTP reverse-path rewritten from by casper.infradead.org. See http://www.infradead.org/rpr.html Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Only bfq needs to code to track icq, so make it conditional. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara --- block/Kconfig | 3 ++ block/Kconfig.iosched | 1 + block/blk-ioc.c | 64 ++++++++++++++++++++++++--------------- block/blk.h | 6 ++++ include/linux/iocontext.h | 6 ++-- 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/block/Kconfig b/block/Kconfig index c6ce41a5e5b27..d5d4197b7ed2d 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -35,6 +35,9 @@ config BLK_CGROUP_RWSTAT config BLK_DEV_BSG_COMMON tristate +config BLK_ICQ + bool + config BLK_DEV_BSGLIB bool "Block layer SG support v4 helper lib" select BLK_DEV_BSG_COMMON diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched index 885fee86dfcae..6155161460862 100644 --- a/block/Kconfig.iosched +++ b/block/Kconfig.iosched @@ -18,6 +18,7 @@ config MQ_IOSCHED_KYBER config IOSCHED_BFQ tristate "BFQ I/O scheduler" + select BLK_ICQ help BFQ I/O scheduler for BLK-MQ. BFQ distributes the bandwidth of of the device among all processes according to their weights, diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 32ae006e1b3e8..5f99b9c833328 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -30,6 +30,7 @@ static void get_io_context(struct io_context *ioc) atomic_long_inc(&ioc->refcount); } +#ifdef CONFIG_BLK_ICQ static void icq_free_icq_rcu(struct rcu_head *head) { struct io_cq *icq = container_of(head, struct io_cq, __rcu_head); @@ -161,6 +162,40 @@ static bool ioc_delay_free(struct io_context *ioc) return false; } +/** + * ioc_clear_queue - break any ioc association with the specified queue + * @q: request_queue being cleared + * + * Walk @q->icq_list and exit all io_cq's. + */ +void ioc_clear_queue(struct request_queue *q) +{ + LIST_HEAD(icq_list); + + spin_lock_irq(&q->queue_lock); + list_splice_init(&q->icq_list, &icq_list); + spin_unlock_irq(&q->queue_lock); + + while (!list_empty(&icq_list)) { + struct io_cq *icq = + list_entry(icq_list.next, struct io_cq, q_node); + + spin_lock_irq(&icq->ioc->lock); + if (!(icq->flags & ICQ_DESTROYED)) + ioc_destroy_icq(icq); + spin_unlock_irq(&icq->ioc->lock); + } +} +#else /* CONFIG_BLK_ICQ */ +static inline void ioc_exit_icqs(struct io_context *ioc) +{ +} +static inline bool ioc_delay_free(struct io_context *ioc) +{ + return false; +} +#endif /* CONFIG_BLK_ICQ */ + /** * put_io_context - put a reference of io_context * @ioc: io_context to put @@ -192,31 +227,6 @@ void exit_io_context(struct task_struct *task) } } -/** - * ioc_clear_queue - break any ioc association with the specified queue - * @q: request_queue being cleared - * - * Walk @q->icq_list and exit all io_cq's. - */ -void ioc_clear_queue(struct request_queue *q) -{ - LIST_HEAD(icq_list); - - spin_lock_irq(&q->queue_lock); - list_splice_init(&q->icq_list, &icq_list); - spin_unlock_irq(&q->queue_lock); - - while (!list_empty(&icq_list)) { - struct io_cq *icq = - list_entry(icq_list.next, struct io_cq, q_node); - - spin_lock_irq(&icq->ioc->lock); - if (!(icq->flags & ICQ_DESTROYED)) - ioc_destroy_icq(icq); - spin_unlock_irq(&icq->ioc->lock); - } -} - static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) { struct io_context *ioc; @@ -228,10 +238,12 @@ static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) atomic_long_set(&ioc->refcount, 1); atomic_set(&ioc->active_ref, 1); +#ifdef CONFIG_BLK_ICQ spin_lock_init(&ioc->lock); INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC); INIT_HLIST_HEAD(&ioc->icq_list); INIT_WORK(&ioc->release_work, ioc_release_fn); +#endif return ioc; } @@ -316,6 +328,7 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) return 0; } +#ifdef CONFIG_BLK_ICQ /** * ioc_lookup_icq - lookup io_cq from ioc * @q: the associated request_queue @@ -441,3 +454,4 @@ static int __init blk_ioc_init(void) return 0; } subsys_initcall(blk_ioc_init); +#endif /* CONFIG_BLK_ICQ */ diff --git a/block/blk.h b/block/blk.h index a55d82c3d1c21..39e822537d1a8 100644 --- a/block/blk.h +++ b/block/blk.h @@ -365,7 +365,13 @@ static inline unsigned int bio_aligned_discard_max_sectors( */ struct io_cq *ioc_find_get_icq(struct request_queue *q); struct io_cq *ioc_lookup_icq(struct request_queue *q); +#ifdef CONFIG_BLK_ICQ void ioc_clear_queue(struct request_queue *q); +#else +static inline void ioc_clear_queue(struct request_queue *q) +{ +} +#endif /* CONFIG_BLK_ICQ */ #ifdef CONFIG_BLK_DEV_THROTTLING_LOW extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page); diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 82c7f4f5f4f59..ef98a994b7b2e 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -100,16 +100,18 @@ struct io_context { atomic_long_t refcount; atomic_t active_ref; + unsigned short ioprio; + +#ifdef CONFIG_BLK_ICQ /* all the fields below are protected by this lock */ spinlock_t lock; - unsigned short ioprio; - struct radix_tree_root icq_tree; struct io_cq __rcu *icq_hint; struct hlist_head icq_list; struct work_struct release_work; +#endif /* CONFIG_BLK_ICQ */ }; struct task_struct;