From patchwork Thu Jun 16 09:21:57 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883634 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 530DBC43334 for ; Thu, 16 Jun 2022 09:22:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1359505AbiFPJWt (ORCPT ); Thu, 16 Jun 2022 05:22:49 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47318 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231648AbiFPJWs (ORCPT ); Thu, 16 Jun 2022 05:22:48 -0400 Received: from mail-wr1-x42f.google.com (mail-wr1-x42f.google.com [IPv6:2a00:1450:4864:20::42f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2218E11174 for ; Thu, 16 Jun 2022 02:22:45 -0700 (PDT) Received: by mail-wr1-x42f.google.com with SMTP id w17so1041105wrg.7 for ; Thu, 16 Jun 2022 02:22:45 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=u0zSyTW9TLDpn72Kx5d9duay8y2dPXlfqcu88lqUDF4=; b=EcISHViODwoq+EyGJDKtcG0B09NTzVc/taDb42qaGsVGq9wm1BT9VVhTvRuqWvDCzU GRReTWMh7Z2cZavVWAy0Ke0Pf2Y75Vo3eBAX+T2ewHSsaurRI4bYydKR292/mhzfKDlD Zv0772tBG3aBxalPZ9lchHO8ltjutcYzwAEaCafQ7jayAySXOaHW3HPYjb4aMuiBQrR1 /HzYV+X03nOtzY3ekXU6Hm7h6xlzMA/FRpuI9MSAVyRkOtz+VxyfIR6fxWWGIauqdYtL y9QXQAmt2jO5/nPf8tW/wZ9aE/tfq/2QTsmsO9HZSdf8OQWbuUyGdEFjjnJwyOD6ko2h ijig== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=u0zSyTW9TLDpn72Kx5d9duay8y2dPXlfqcu88lqUDF4=; b=c1TX7twGd+odRYGl7Ph28ubf1kLGq5BWsz9BUdtwfQGQBj80D4iIZhiltQQFgY69bV KiKeHrslQ8aK3gAxx7w+X7SWJGQamUqySIStb6Z+JzqOo9Q/JKqd93LPiQhPvKhtXMSn W+MnB8Eq7GXbJcTTT7Zk2MCEvI/Kqb8nKgoAE83ab8igAfGZGn9tcttbQO1/JuMEUQt5 8XLhF6CNdIy0oWNKJ6XLi/maSeMW9OjEsY4bHLGGEOPajxEjcZaEO/AkFNMN2Jd6ywad d4qMB0PPEvqjuU0cVxvi0xh2/7tXqnwJg+DvsT9e0j6tgusnM7+SXNF8JQFS8WNLL3QU nihg== X-Gm-Message-State: AJIora8M6YRedLKYG9SOc8zyZcYcLLbf8qBOg+BVTb3zIXdgC7fHrSHx jVadpm1524EgtdLKdyKiUioGVfIH+bka9w== X-Google-Smtp-Source: AGRyM1ucezeYzxbqo9L5opI51RediPrM4Wt6V8YRIpQNDc/OjLBJWJGhCekbzvMlwddOUCuqKoYqeA== X-Received: by 2002:adf:fb0b:0:b0:21a:b15:6013 with SMTP id c11-20020adffb0b000000b0021a0b156013mr3608840wrr.268.1655371363799; Thu, 16 Jun 2022 02:22:43 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.42 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:43 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 01/16] io_uring: rw: delegate sync completions to core io_uring Date: Thu, 16 Jun 2022 10:21:57 +0100 Message-Id: <32ef005b45d23bf6b5e6837740dc0331bb051bd4.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org io_issue_sqe() from the io_uring core knows how to complete requests based on the returned error code, we can delegate io_read()/io_write() completion to it. Make kiocb_done() to return the right completion code and propagate it. Signed-off-by: Pavel Begunkov --- io_uring/rw.c | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/io_uring/rw.c b/io_uring/rw.c index fa1063c738f8..818692a83d75 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -207,15 +207,6 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) return false; } -static void __io_complete_rw(struct io_kiocb *req, long res, - unsigned int issue_flags) -{ - if (__io_complete_rw_common(req, res)) - return; - io_req_set_res(req, req->cqe.res, io_put_kbuf(req, issue_flags)); - __io_req_complete(req, issue_flags); -} - static void io_complete_rw(struct kiocb *kiocb, long res) { struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); @@ -247,7 +238,7 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) smp_store_release(&req->iopoll_completed, 1); } -static void kiocb_done(struct io_kiocb *req, ssize_t ret, +static int kiocb_done(struct io_kiocb *req, ssize_t ret, unsigned int issue_flags) { struct io_async_rw *io = req->async_data; @@ -263,10 +254,15 @@ static void kiocb_done(struct io_kiocb *req, ssize_t ret, if (req->flags & REQ_F_CUR_POS) req->file->f_pos = rw->kiocb.ki_pos; - if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) - __io_complete_rw(req, ret, issue_flags); - else + if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) { + if (!__io_complete_rw_common(req, ret)) { + io_req_set_res(req, req->cqe.res, + io_put_kbuf(req, issue_flags)); + return IOU_OK; + } + } else { io_rw_done(&rw->kiocb, ret); + } if (req->flags & REQ_F_REISSUE) { req->flags &= ~REQ_F_REISSUE; @@ -275,6 +271,7 @@ static void kiocb_done(struct io_kiocb *req, ssize_t ret, else io_req_task_queue_fail(req, ret); } + return IOU_ISSUE_SKIP_COMPLETE; } static int __io_import_fixed(struct io_kiocb *req, int ddir, @@ -846,7 +843,9 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) goto done; ret = 0; } else if (ret == -EIOCBQUEUED) { - goto out_free; + if (iovec) + kfree(iovec); + return IOU_ISSUE_SKIP_COMPLETE; } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock || (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) { /* read all, failed, already did sync or don't want to retry */ @@ -904,12 +903,10 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) iov_iter_restore(&s->iter, &s->iter_state); } while (ret > 0); done: - kiocb_done(req, ret, issue_flags); -out_free: /* it's faster to check here then delegate to kfree */ if (iovec) kfree(iovec); - return IOU_ISSUE_SKIP_COMPLETE; + return kiocb_done(req, ret, issue_flags); } int io_write(struct io_kiocb *req, unsigned int issue_flags) @@ -959,8 +956,10 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags) ppos = io_kiocb_update_pos(req); ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res); - if (unlikely(ret)) - goto out_free; + if (unlikely(ret)) { + kfree(iovec); + return ret; + } /* * Open-code file_start_write here to grab freeze protection, @@ -1002,15 +1001,13 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags) if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) goto copy_iov; done: - kiocb_done(req, ret2, issue_flags); - ret = IOU_ISSUE_SKIP_COMPLETE; + ret = kiocb_done(req, ret2, issue_flags); } else { copy_iov: iov_iter_restore(&s->iter, &s->iter_state); ret = io_setup_async_rw(req, iovec, s, false); return ret ?: -EAGAIN; } -out_free: /* it's reportedly faster than delegating the null check to kfree() */ if (iovec) kfree(iovec); From patchwork Thu Jun 16 09:21:58 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883635 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id EFFA3CCA47A for ; Thu, 16 Jun 2022 09:22:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231648AbiFPJWt (ORCPT ); Thu, 16 Jun 2022 05:22:49 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47326 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231664AbiFPJWs (ORCPT ); Thu, 16 Jun 2022 05:22:48 -0400 Received: from mail-wr1-x42b.google.com (mail-wr1-x42b.google.com [IPv6:2a00:1450:4864:20::42b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id C50C611167 for ; Thu, 16 Jun 2022 02:22:46 -0700 (PDT) Received: by mail-wr1-x42b.google.com with SMTP id s1so1035718wra.9 for ; Thu, 16 Jun 2022 02:22:46 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=0ZsaHpJ+wHbQ542pIhquFY7dETyZEEfQcuRQQLy5A28=; b=Ne2qDU/PEqXHQeBk7GF3dMp6i/rEBKurv9RqgtWdBWqfRoiC/OSAq4/DPVbYHtoOsA svLLQy66KYqVoDd4l9oEBkuBmKnETzAZkM8V+VhZPRCC3lrNpRm68TQIMmLqw/xIJIuZ j6cXC12H6mI/j4/p3wlRiHYIHPHlQnvI5ZkB62tiHZROmU7wnt6vdGaNwFgDiQSGGTIP lfAR3yfGnv9bUVMLIAQ24AE4FLvgqlb1r4f8SmyqRbFv3H3RMaWwrGvSq63G1a/3q+5Q vOOmfsIZYZH6rbwHtwmUQYxZ2MSYuth/CYmiwzlwKlStom4mqHPI9XM78pa6njQk9Oz2 +N0w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=0ZsaHpJ+wHbQ542pIhquFY7dETyZEEfQcuRQQLy5A28=; b=U0EJENDcJjKuSBBBraiVg+OYodBCgNgGrLPLlJeBsf7ht0ZBtabJrFVFLXaVKCRBZ3 Upwtni1APjchpeldzJ5FaVMrI5I/o8dmN2Qu7hHLj533s/WUjA9vMzZ8h9raPTNO0SOj 2V4Po7Vnj0DZxxPwgqlGUFzh7mjuaASiDDhDx642XV68+4OUZuEYv5ScJBMSOmxHjhTI /11XQ463BQR4e8QN6OaN5mePkrZaAbxOds2X/6x0xyZEgSuT7ozXElYjU490hASXR2hQ qPtQXezGuCxqLsDazfZJs4UtRieYUJkwxo7ksLdNFoJAAzFBgeecNVEJ0P96yplrAY1D yL5A== X-Gm-Message-State: AJIora/E+IRZ994uy7J+ZuKPnKUXZxa+ZNzvzT666CfqGJk9TIOfV954 amIQdbe2cXCEJ1O3FO3x/XDIhVcQnuSVMg== X-Google-Smtp-Source: AGRyM1tXNEK//2jWAerbkkagqbLN2PkzYsLgGHE2n+NOw1sjZhpnW12wEYHkxQ9AkdzOzrlBS9xFaw== X-Received: by 2002:a05:6000:186e:b0:218:5f5d:9c55 with SMTP id d14-20020a056000186e00b002185f5d9c55mr3562733wri.128.1655371365004; Thu, 16 Jun 2022 02:22:45 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.43 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:44 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 02/16] io_uring: kill REQ_F_COMPLETE_INLINE Date: Thu, 16 Jun 2022 10:21:58 +0100 Message-Id: <600ba20a9338b8a39b249b23d3d177803613dde4.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org REQ_F_COMPLETE_INLINE is only needed to delay queueing into the completion list to io_queue_sqe() as __io_req_complete() is inlined and we don't want to bloat the kernel. As now we complete in a more centralised fashion in io_issue_sqe() we can get rid of the flag and queue to the list directly. Signed-off-by: Pavel Begunkov --- io_uring/io_uring.c | 18 +++++++----------- io_uring/io_uring.h | 5 ----- io_uring/io_uring_types.h | 3 --- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index f1ecfdd0166e..02a70e7eb774 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -742,10 +742,7 @@ void io_req_complete_post(struct io_kiocb *req) inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags) { - if (issue_flags & IO_URING_F_COMPLETE_DEFER) - req->flags |= REQ_F_COMPLETE_INLINE; - else - io_req_complete_post(req); + io_req_complete_post(req); } void io_req_complete_failed(struct io_kiocb *req, s32 res) @@ -1581,9 +1578,12 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) if (creds) revert_creds(creds); - if (ret == IOU_OK) - __io_req_complete(req, issue_flags); - else if (ret != IOU_ISSUE_SKIP_COMPLETE) + if (ret == IOU_OK) { + if (issue_flags & IO_URING_F_COMPLETE_DEFER) + io_req_add_compl_list(req); + else + io_req_complete_post(req); + } else if (ret != IOU_ISSUE_SKIP_COMPLETE) return ret; /* If the op doesn't have a file, we're not polling for it */ @@ -1748,10 +1748,6 @@ static inline void io_queue_sqe(struct io_kiocb *req) ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); - if (req->flags & REQ_F_COMPLETE_INLINE) { - io_req_add_compl_list(req); - return; - } /* * We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index e30e639c2822..3f6cad3d356c 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -217,11 +217,6 @@ static inline bool io_run_task_work(void) return false; } -static inline void io_req_complete_state(struct io_kiocb *req) -{ - req->flags |= REQ_F_COMPLETE_INLINE; -} - static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) { if (!*locked) { diff --git a/io_uring/io_uring_types.h b/io_uring/io_uring_types.h index ef1cf86e8932..4576ea8cad2e 100644 --- a/io_uring/io_uring_types.h +++ b/io_uring/io_uring_types.h @@ -301,7 +301,6 @@ enum { REQ_F_POLLED_BIT, REQ_F_BUFFER_SELECTED_BIT, REQ_F_BUFFER_RING_BIT, - REQ_F_COMPLETE_INLINE_BIT, REQ_F_REISSUE_BIT, REQ_F_CREDS_BIT, REQ_F_REFCOUNT_BIT, @@ -356,8 +355,6 @@ enum { REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), /* buffer selected from ring, needs commit */ REQ_F_BUFFER_RING = BIT(REQ_F_BUFFER_RING_BIT), - /* completion is deferred through io_comp_state */ - REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT), /* caller should reissue async */ REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT), /* supports async reads/writes */ From patchwork Thu Jun 16 09:21:59 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883636 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7B085C433EF for ; Thu, 16 Jun 2022 09:22:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231664AbiFPJWu (ORCPT ); Thu, 16 Jun 2022 05:22:50 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47342 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1359481AbiFPJWt (ORCPT ); Thu, 16 Jun 2022 05:22:49 -0400 Received: from mail-wr1-x436.google.com (mail-wr1-x436.google.com [IPv6:2a00:1450:4864:20::436]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id C4C3411A10 for ; Thu, 16 Jun 2022 02:22:47 -0700 (PDT) Received: by mail-wr1-x436.google.com with SMTP id m24so1030919wrb.10 for ; Thu, 16 Jun 2022 02:22:47 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=BCuEENWz7BFINBG8zBMSUF8mtkwrlLjt3CzidM5Dw40=; b=VdouYoSc1tjxSY04emPi7DQXhRnPkD3YPEQ+Gwl4I7KKBW+W46G8cVzN9wpu4QP6Oa TRritE6UvqYEAidxLw2xIkQTwtyYpyyBkIGwAiExIK/SFTJ4Cy/ikJflLk/zhTUhAL2D yJchwGFwFe3LzRN+hYKo2fygzd0g13pt2ypeQq99ieTwvpyGThTh/Bm1BhahAXdgNeBs LeT68Lx9W+9ojnZCecruqA66lHK1fHITSf1atXwLzRjHeC0N9eC5IXj30nUJJAEHLbkZ nlN3o3+NpqgQleg3QkKb9NFwyEHejEUo5+4wVgv1RzixW5LB9YBrRRA0xYVIVtq1PvVD b8UA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=BCuEENWz7BFINBG8zBMSUF8mtkwrlLjt3CzidM5Dw40=; b=cQ6BxIK1bWnOHPeWPP10pHxIG2/ATzlFaOcPVSVQq3gQ01PyimlQN/Hd1abmoWpoo0 UYmbio7guamAu0Y+9DOxw0CrScF8fDCTyjk8eALc9Yn5DLwYOzRjigamJ1KWjhMYVauS ue6imAep0UwgMyDpgBYyC/BX4zmpHhWYRMrqs024YIfyJAKDHJkehd1xtDwOxhGozoVH yGQ91qAPGF+XZu7UYDxFD2C2LklohDYziTfxSdy/+tnv0eUMRo3QQG1OPXlVXhKsGPcv bsfTxjFrA6r/+6HX8XL6ng2Kcb8sUJE4pkbFSJ8aoAIjGVmsjaJlPkokoWPgZGGGrj3o jQzg== X-Gm-Message-State: AJIora8NWADNu424zzoLXOwbNazQqkO0Rv9ppQm4Bne5JExj0jlS4OJ4 lt+eSDEnptizWCZYN7k6XyeEA6aGhRIqvw== X-Google-Smtp-Source: AGRyM1sD4pti1RXsrKBBSK2MOt7M1yGWkRMSfOU9b+okooa4hbDtlEiCUMj0kuPllqfK5v2GmcpWpA== X-Received: by 2002:adf:ed45:0:b0:210:2f9c:f269 with SMTP id u5-20020adfed45000000b002102f9cf269mr3822602wro.470.1655371366073; Thu, 16 Jun 2022 02:22:46 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.45 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:45 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 03/16] io_uring: refactor io_req_task_complete() Date: Thu, 16 Jun 2022 10:21:59 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Clean up io_req_task_complete() and deduplicate io_put_kbuf() calls. Signed-off-by: Pavel Begunkov --- io_uring/io_uring.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 02a70e7eb774..1bcd2a8ebd4c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1306,15 +1306,19 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) return ret; } -inline void io_req_task_complete(struct io_kiocb *req, bool *locked) + +void io_req_task_complete(struct io_kiocb *req, bool *locked) { - if (*locked) { - req->cqe.flags |= io_put_kbuf(req, 0); + if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { + unsigned issue_flags = *locked ? 0 : IO_URING_F_UNLOCKED; + + req->cqe.flags |= io_put_kbuf(req, issue_flags); + } + + if (*locked) io_req_add_compl_list(req); - } else { - req->cqe.flags |= io_put_kbuf(req, IO_URING_F_UNLOCKED); + else io_req_complete_post(req); - } } /* From patchwork Thu Jun 16 09:22:00 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883637 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 94374C43334 for ; Thu, 16 Jun 2022 09:22:52 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1359481AbiFPJWv (ORCPT ); Thu, 16 Jun 2022 05:22:51 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47400 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376598AbiFPJWu (ORCPT ); Thu, 16 Jun 2022 05:22:50 -0400 Received: from mail-wm1-x332.google.com (mail-wm1-x332.google.com [IPv6:2a00:1450:4864:20::332]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 0D7E211174 for ; Thu, 16 Jun 2022 02:22:48 -0700 (PDT) Received: by mail-wm1-x332.google.com with SMTP id a10so422660wmj.5 for ; Thu, 16 Jun 2022 02:22:48 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=+oR3rNXZgU1ixzRI7JxN2AwlqFap+l1SjK5ceoxKnGA=; b=BwAdIGhAhpc7c501cFHLJotqPAyCzl6wkmPjxgsOnqKqo6F6kEje8zJHT5kcObE8FD Buu+qfSy9D9VEvXN3F9x3p1ixGCZU2+M7cs4S3rqcfxhToxXYiSYGwIfizNZy03hbQS8 o+Ek7uX/l854KRgrhxAhzDd+U+VJFvTC6lXu6bIMor7jsoZvgAAud7IFJ6npAnGzB4yB DEDEpB3pStbqJ9y+1P4WpOj0o/O2vc/PcRfOtN6mrTxbBImxUCC9B+OwsxtQfJkevz6W e0H3XYZG/j+R8Kwg8oQc+fUQLjFvSCF/20wXPAPNvAjVRsHNMaplGzrXN2pFJ7gS02+G bPUA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=+oR3rNXZgU1ixzRI7JxN2AwlqFap+l1SjK5ceoxKnGA=; b=lDRN+PTvfPA1fAbTj6j1H0oEaR4XdXqlwWdKO5PxZXNRuFCnZSIPU16YipEJbYnHgq h9fn8G5THZA0MHphTZdK/bRSXmU0bWNnCPGpzivsyZqd0MY3J97TpQw2dDq+0h2dFiHb W+S/4ZamV+0xtCIOsjn/8/EWcetpqqprwQaglZODfm3rbWz6H8hTQ9wtK+XZ9XZyL6PG LJoghzZaYWgjqkBYK8Dk+FBzAP7uav1Leq2uXAHZa2cXV+VBZsyYxBkpxnpUSloUEm/M 65vsGM1QYegUeSI3RvqTENKAxwRxdZh4YBzgQVX1JDudwiTLXIak2WIGXyMtC2Jk71v/ DhfA== X-Gm-Message-State: AOAM530OGcPAWeZmc96mvKK7/YGPHQxmBEtxzHa50w7PpGFyqi94dit+ VVDyTSt168PMfhc2CL8/sFwYW0mWMFhrYA== X-Google-Smtp-Source: ABdhPJwR2kkyCWmw0lVe9Oxc+RMP0YzPJG7P2+tVLDkV33mFEb41aT+tnP6UpHd2iUqVKX9HFvbPYg== X-Received: by 2002:a05:600c:348d:b0:39c:652b:5153 with SMTP id a13-20020a05600c348d00b0039c652b5153mr14575342wmq.24.1655371367081; Thu, 16 Jun 2022 02:22:47 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.46 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:46 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 04/16] io_uring: don't inline io_put_kbuf Date: Thu, 16 Jun 2022 10:22:00 +0100 Message-Id: <2e21ccf0be471ffa654032914b9430813cae53f8.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org io_put_kbuf() is huge, don't bloat the kernel with inlining. Signed-off-by: Pavel Begunkov --- io_uring/kbuf.c | 33 +++++++++++++++++++++++++++++++++ io_uring/kbuf.h | 38 ++++++-------------------------------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 5885343705bd..223d9db2ba94 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -82,6 +82,39 @@ static int io_buffer_add_list(struct io_ring_ctx *ctx, return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL)); } +unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags) +{ + unsigned int cflags; + + /* + * We can add this buffer back to two lists: + * + * 1) The io_buffers_cache list. This one is protected by the + * ctx->uring_lock. If we already hold this lock, add back to this + * list as we can grab it from issue as well. + * 2) The io_buffers_comp list. This one is protected by the + * ctx->completion_lock. + * + * We migrate buffers from the comp_list to the issue cache list + * when we need one. + */ + if (req->flags & REQ_F_BUFFER_RING) { + /* no buffers to recycle for this case */ + cflags = __io_put_kbuf_list(req, NULL); + } else if (issue_flags & IO_URING_F_UNLOCKED) { + struct io_ring_ctx *ctx = req->ctx; + + spin_lock(&ctx->completion_lock); + cflags = __io_put_kbuf_list(req, &ctx->io_buffers_comp); + spin_unlock(&ctx->completion_lock); + } else { + lockdep_assert_held(&req->ctx->uring_lock); + + cflags = __io_put_kbuf_list(req, &req->ctx->io_buffers_cache); + } + return cflags; +} + static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len, struct io_buffer_list *bl) { diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index 80b6df2c7535..5da3d4039aed 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -47,6 +47,8 @@ int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags); int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg); int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg); +unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags); + static inline bool io_do_buffer_select(struct io_kiocb *req) { if (!(req->flags & REQ_F_BUFFER_SELECT)) @@ -70,7 +72,8 @@ static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) __io_kbuf_recycle(req, issue_flags); } -static unsigned int __io_put_kbuf(struct io_kiocb *req, struct list_head *list) +static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req, + struct list_head *list) { if (req->flags & REQ_F_BUFFER_RING) { if (req->buf_list) @@ -90,44 +93,15 @@ static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req) if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) return 0; - return __io_put_kbuf(req, &req->ctx->io_buffers_comp); + return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp); } static inline unsigned int io_put_kbuf(struct io_kiocb *req, unsigned issue_flags) { - unsigned int cflags; if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) return 0; - - /* - * We can add this buffer back to two lists: - * - * 1) The io_buffers_cache list. This one is protected by the - * ctx->uring_lock. If we already hold this lock, add back to this - * list as we can grab it from issue as well. - * 2) The io_buffers_comp list. This one is protected by the - * ctx->completion_lock. - * - * We migrate buffers from the comp_list to the issue cache list - * when we need one. - */ - if (req->flags & REQ_F_BUFFER_RING) { - /* no buffers to recycle for this case */ - cflags = __io_put_kbuf(req, NULL); - } else if (issue_flags & IO_URING_F_UNLOCKED) { - struct io_ring_ctx *ctx = req->ctx; - - spin_lock(&ctx->completion_lock); - cflags = __io_put_kbuf(req, &ctx->io_buffers_comp); - spin_unlock(&ctx->completion_lock); - } else { - lockdep_assert_held(&req->ctx->uring_lock); - - cflags = __io_put_kbuf(req, &req->ctx->io_buffers_cache); - } - - return cflags; + return __io_put_kbuf(req, issue_flags); } #endif From patchwork Thu Jun 16 09:22:01 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883638 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 53804CCA47A for ; Thu, 16 Jun 2022 09:22:53 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1359793AbiFPJWw (ORCPT ); Thu, 16 Jun 2022 05:22:52 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47426 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376655AbiFPJWu (ORCPT ); Thu, 16 Jun 2022 05:22:50 -0400 Received: from mail-wm1-x333.google.com (mail-wm1-x333.google.com [IPv6:2a00:1450:4864:20::333]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D143813F8F for ; Thu, 16 Jun 2022 02:22:49 -0700 (PDT) Received: by mail-wm1-x333.google.com with SMTP id i81-20020a1c3b54000000b0039c76434147so2500284wma.1 for ; Thu, 16 Jun 2022 02:22:49 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=xtoDyDvNXPTaclwDuSrBJ25RRT45S7N/gTULdXm19LA=; b=PZHLxxb96ZZSjv9zS9xYoL2jHEbnmGHauRivawSzqNRAZFKvp6BUol0xZz+lQc4HLg 0aiy2bYHX7wHBQ9nBLIjeULxGqryfTFA3hu5voUe4DGhN1wNvKBQsXW8qRfci5yVLTE/ 7whO7Y4p2zBmo8+JgM/kNqw2cSMLTJ1yZufeVIjD62wz2pQUEN3A49pRbswd/6YQhHK2 q63BvejoRjeLEDUxrmq1Ex/OISR0W8bR2TtA5auv0OnTqum1eTMcQ4JxkaogFQEB+Iug rT3vYxf2it4k3RWuJ+0nPx4SDBgOKL496AlYtna963vnDrvoisLsBOvJGt+Cs8OXfJ+l rzZw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=xtoDyDvNXPTaclwDuSrBJ25RRT45S7N/gTULdXm19LA=; b=VOFDAgCUtWt1l70t5vqYh5+2+y2h8/PElY1gBh80+LRZkLmBdeMkHCTRQ/e8Qj8aYW TMg1KJNGo/ogovchuW3hpHDRfDJHYkOGE5Acdakh8lgltw6bIl06Lw45SPjSaEsvkh3U O3YjHLYvN/TctS1/1gAjhLHZw5VMH0ZJv2VYZdx2Zm3csLaDEs4PUtGL3McVuEV03Yms Pvgzh+CiDKfwHxmqRGpuWr3QGZtlMefI80KzXse2zVohX/2YCAKoyoiRgJy5+NRYQIcB nTgvZzQ23KX1KJ3Xl/7SzWi6dDCac1JX4TK684Sdy4FZ0cn/eaSycGSQbTmpMQglA/i8 X0iw== X-Gm-Message-State: AJIora9Lys6fziA6M91hbC2QP7eIZsyVGYS1mPO3TQJZ0pvS14ZMIQid KSs9YGZv9LQPeMFY/fTDXOoKEpEMtGgYhg== X-Google-Smtp-Source: AGRyM1sUE0TRlFkQNVACxK/gvFWwFzyx1ANrJhilWlWHKEJ/I8T8hH1OqoeHUT58iFRmTEsdTsPDSg== X-Received: by 2002:a7b:c4d4:0:b0:39c:5bb7:2210 with SMTP id g20-20020a7bc4d4000000b0039c5bb72210mr3867363wmk.99.1655371368197; Thu, 16 Jun 2022 02:22:48 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.47 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:47 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com, Hao Xu Subject: [PATCH for-next v3 05/16] io_uring: poll: remove unnecessary req->ref set Date: Thu, 16 Jun 2022 10:22:01 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org From: Hao Xu We now don't need to set req->refcount for poll requests since the reworked poll code ensures no request release race. Signed-off-by: Hao Xu Signed-off-by: Pavel Begunkov --- io_uring/poll.c | 1 - 1 file changed, 1 deletion(-) diff --git a/io_uring/poll.c b/io_uring/poll.c index 0df5eca93b16..73584c4e3e9b 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -683,7 +683,6 @@ int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP)) return -EINVAL; - io_req_set_refcount(req); req->apoll_events = poll->events = io_poll_parse_events(sqe, flags); return 0; } From patchwork Thu Jun 16 09:22:02 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883640 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B8BBEC433EF for ; Thu, 16 Jun 2022 09:22:55 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376550AbiFPJWy (ORCPT ); Thu, 16 Jun 2022 05:22:54 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47518 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376481AbiFPJWx (ORCPT ); Thu, 16 Jun 2022 05:22:53 -0400 Received: from mail-wr1-x42b.google.com (mail-wr1-x42b.google.com [IPv6:2a00:1450:4864:20::42b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2B77311174 for ; Thu, 16 Jun 2022 02:22:51 -0700 (PDT) Received: by mail-wr1-x42b.google.com with SMTP id o8so1053920wro.3 for ; Thu, 16 Jun 2022 02:22:51 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=TlwmWSRz80ue7aucZVOFLkddfuJsrhKYSVrtCD5ocd0=; b=N8IkazEb0RgeDajtUvEIVYLOHH+ZyhQmRdCX2YnGg3VztE1swwstnI4fxBL9VrsG3l fKFA2nqfCyKD95piBs7zkShRAgwJahzXJfDDRCo4gONFD3A8BkMWzccZxWdhw+shvsuy JeAzO6AsQFoevuWlyN7q2Uew/TVOXyda70pYZNBUHA5c224hPQVDmSwltU1X68kkw4KZ PejuStAC5/wepIJAJf7Vl/ubQHIawq4yphpBACxciAzDp3xdWyBg0fwSxtqqZQzpxHq2 xietv6rJsJjGQWuaurLXnI0yIIxVHIPTiUB1WMKfjNMOHG6mjXm1sHClR3ka8pKcvaPT Vhyg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=TlwmWSRz80ue7aucZVOFLkddfuJsrhKYSVrtCD5ocd0=; b=6PIbacCaqqmISNuJMN3pKQSXjyLXPyzPVWKs3/BAMvV8kNHrPDS+Cs7r2EOwTVbf0l VwXHcO5H+juONtrXhG1uL29UVQIos10nSuri1pnB9gt7dUa1gfrQzxPO7On3QG7oN+4P 97fLZkfNoMoiWZDqlCad8FMVi8QB08ykjCr5Vlo2y0ES1uRAq01Nf5qjIeumAygF/lYh S2dL+lvv/6H8DmoR6npSo+bp1yg4FO8roRSJzUfgTuWLKoz7yKTpM2FVl4sCVRAjLCgq q5crzuKmqMzbrlHyCmUgkpi1iJbh/k4SFhIUU6Uc+jtwk7yfT6PF5NXsgtDkw+UDRk2l JDKg== X-Gm-Message-State: AJIora9R2oNlV3pgeSOZl9mDTyHx1kib/uzmy65PuSdC5wmivr7PEWGJ EGF7HUyA8Fac1p7AkOh9ITXEhEZSCFdWLw== X-Google-Smtp-Source: AGRyM1vYM9Z5kXFaLdWnoTrdQRIrtznj+3/qCox8QaxGA+2/m9VEoPjg/EPKBz3KpeaLYDRRHTGzCQ== X-Received: by 2002:adf:da49:0:b0:21a:18af:ab33 with SMTP id r9-20020adfda49000000b0021a18afab33mr3714671wrl.645.1655371369307; Thu, 16 Jun 2022 02:22:49 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.48 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:48 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com, Hao Xu Subject: [PATCH for-next v3 06/16] io_uring: switch cancel_hash to use per entry spinlock Date: Thu, 16 Jun 2022 10:22:02 +0100 Message-Id: <05d1e135b0c8bce9d1441e6346776589e5783e26.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org From: Hao Xu Add a new io_hash_bucket structure so that each bucket in cancel_hash has separate spinlock. Use per entry lock for cancel_hash, this removes some completion lock invocation and remove contension between different cancel_hash entries. Signed-off-by: Hao Xu Signed-off-by: Pavel Begunkov --- io_uring/cancel.c | 14 ++++++- io_uring/cancel.h | 6 +++ io_uring/fdinfo.c | 9 +++-- io_uring/io_uring.c | 9 +++-- io_uring/io_uring_types.h | 2 +- io_uring/poll.c | 80 ++++++++++++++++++++++++--------------- 6 files changed, 80 insertions(+), 40 deletions(-) diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 83cceb52d82d..6f2888388a40 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -93,14 +93,14 @@ int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd) if (!ret) return 0; - spin_lock(&ctx->completion_lock); ret = io_poll_cancel(ctx, cd); if (ret != -ENOENT) goto out; + spin_lock(&ctx->completion_lock); if (!(cd->flags & IORING_ASYNC_CANCEL_FD)) ret = io_timeout_cancel(ctx, cd); -out: spin_unlock(&ctx->completion_lock); +out: return ret; } @@ -192,3 +192,13 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) io_req_set_res(req, ret, 0); return IOU_OK; } + +void init_hash_table(struct io_hash_bucket *hash_table, unsigned size) +{ + unsigned int i; + + for (i = 0; i < size; i++) { + spin_lock_init(&hash_table[i].lock); + INIT_HLIST_HEAD(&hash_table[i].list); + } +} diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 4f35d8696325..556a7dcf160e 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -4,3 +4,9 @@ int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags); int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd); +void init_hash_table(struct io_hash_bucket *hash_table, unsigned size); + +struct io_hash_bucket { + spinlock_t lock; + struct hlist_head list; +} ____cacheline_aligned_in_smp; diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index fcedde4b4b1e..f941c73f5502 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -13,6 +13,7 @@ #include "io_uring.h" #include "sqpoll.h" #include "fdinfo.h" +#include "cancel.h" #ifdef CONFIG_PROC_FS static __cold int io_uring_show_cred(struct seq_file *m, unsigned int id, @@ -157,17 +158,19 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, mutex_unlock(&ctx->uring_lock); seq_puts(m, "PollList:\n"); - spin_lock(&ctx->completion_lock); for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { - struct hlist_head *list = &ctx->cancel_hash[i]; + struct io_hash_bucket *hb = &ctx->cancel_hash[i]; struct io_kiocb *req; - hlist_for_each_entry(req, list, hash_node) + spin_lock(&hb->lock); + hlist_for_each_entry(req, &hb->list, hash_node) seq_printf(m, " op=%d, task_works=%d\n", req->opcode, task_work_pending(req->task)); + spin_unlock(&hb->lock); } seq_puts(m, "CqOverflowList:\n"); + spin_lock(&ctx->completion_lock); list_for_each_entry(ocqe, &ctx->cq_overflow_list, list) { struct io_uring_cqe *cqe = &ocqe->cqe; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1bcd2a8ebd4c..d0242e5c8d0a 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -89,6 +89,7 @@ #include "fdinfo.h" #include "kbuf.h" #include "rsrc.h" +#include "cancel.h" #include "timeout.h" #include "poll.h" @@ -260,11 +261,13 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) if (hash_bits <= 0) hash_bits = 1; ctx->cancel_hash_bits = hash_bits; - ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head), - GFP_KERNEL); + ctx->cancel_hash = + kmalloc((1U << hash_bits) * sizeof(struct io_hash_bucket), + GFP_KERNEL); if (!ctx->cancel_hash) goto err; - __hash_init(ctx->cancel_hash, 1U << hash_bits); + + init_hash_table(ctx->cancel_hash, 1U << hash_bits); ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL); if (!ctx->dummy_ubuf) diff --git a/io_uring/io_uring_types.h b/io_uring/io_uring_types.h index 4576ea8cad2e..1f8db2dd7af7 100644 --- a/io_uring/io_uring_types.h +++ b/io_uring/io_uring_types.h @@ -224,7 +224,7 @@ struct io_ring_ctx { * manipulate the list, hence no extra locking is needed there. */ struct io_wq_work_list iopoll_list; - struct hlist_head *cancel_hash; + struct io_hash_bucket *cancel_hash; unsigned cancel_hash_bits; bool poll_multi_queue; diff --git a/io_uring/poll.c b/io_uring/poll.c index 73584c4e3e9b..7f6b16f687b0 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -19,6 +19,7 @@ #include "opdef.h" #include "kbuf.h" #include "poll.h" +#include "cancel.h" struct io_poll_update { struct file *file; @@ -73,10 +74,22 @@ static struct io_poll *io_poll_get_single(struct io_kiocb *req) static void io_poll_req_insert(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; - struct hlist_head *list; + u32 index = hash_long(req->cqe.user_data, ctx->cancel_hash_bits); + struct io_hash_bucket *hb = &ctx->cancel_hash[index]; - list = &ctx->cancel_hash[hash_long(req->cqe.user_data, ctx->cancel_hash_bits)]; - hlist_add_head(&req->hash_node, list); + spin_lock(&hb->lock); + hlist_add_head(&req->hash_node, &hb->list); + spin_unlock(&hb->lock); +} + +static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx) +{ + u32 index = hash_long(req->cqe.user_data, ctx->cancel_hash_bits); + spinlock_t *lock = &ctx->cancel_hash[index].lock; + + spin_lock(lock); + hash_del(&req->hash_node); + spin_unlock(lock); } static void io_init_poll_iocb(struct io_poll *poll, __poll_t events, @@ -220,8 +233,8 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) } io_poll_remove_entries(req); + io_poll_req_delete(req, ctx); spin_lock(&ctx->completion_lock); - hash_del(&req->hash_node); req->cqe.flags = 0; __io_req_complete_post(req); io_commit_cqring(ctx); @@ -231,7 +244,6 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) static void io_apoll_task_func(struct io_kiocb *req, bool *locked) { - struct io_ring_ctx *ctx = req->ctx; int ret; ret = io_poll_check_events(req, locked); @@ -239,9 +251,7 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) return; io_poll_remove_entries(req); - spin_lock(&ctx->completion_lock); - hash_del(&req->hash_node); - spin_unlock(&ctx->completion_lock); + io_poll_req_delete(req, req->ctx); if (!ret) io_req_task_submit(req, locked); @@ -435,9 +445,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, return 0; } - spin_lock(&ctx->completion_lock); io_poll_req_insert(req); - spin_unlock(&ctx->completion_lock); if (mask && (poll->events & EPOLLET)) { /* can't multishot if failed, just queue the event we've got */ @@ -534,32 +542,31 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, bool found = false; int i; - spin_lock(&ctx->completion_lock); for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { - struct hlist_head *list; + struct io_hash_bucket *hb = &ctx->cancel_hash[i]; - list = &ctx->cancel_hash[i]; - hlist_for_each_entry_safe(req, tmp, list, hash_node) { + spin_lock(&hb->lock); + hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { if (io_match_task_safe(req, tsk, cancel_all)) { hlist_del_init(&req->hash_node); io_poll_cancel_req(req); found = true; } } + spin_unlock(&hb->lock); } - spin_unlock(&ctx->completion_lock); return found; } static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, struct io_cancel_data *cd) - __must_hold(&ctx->completion_lock) { - struct hlist_head *list; struct io_kiocb *req; + u32 index = hash_long(cd->data, ctx->cancel_hash_bits); + struct io_hash_bucket *hb = &ctx->cancel_hash[index]; - list = &ctx->cancel_hash[hash_long(cd->data, ctx->cancel_hash_bits)]; - hlist_for_each_entry(req, list, hash_node) { + spin_lock(&hb->lock); + hlist_for_each_entry(req, &hb->list, hash_node) { if (cd->data != req->cqe.user_data) continue; if (poll_only && req->opcode != IORING_OP_POLL_ADD) @@ -571,21 +578,21 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, } return req; } + spin_unlock(&hb->lock); return NULL; } static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, struct io_cancel_data *cd) - __must_hold(&ctx->completion_lock) { struct io_kiocb *req; int i; for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { - struct hlist_head *list; + struct io_hash_bucket *hb = &ctx->cancel_hash[i]; - list = &ctx->cancel_hash[i]; - hlist_for_each_entry(req, list, hash_node) { + spin_lock(&hb->lock); + hlist_for_each_entry(req, &hb->list, hash_node) { if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) && req->file != cd->file) continue; @@ -594,12 +601,12 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, req->work.cancel_seq = cd->seq; return req; } + spin_unlock(&hb->lock); } return NULL; } static bool io_poll_disarm(struct io_kiocb *req) - __must_hold(&ctx->completion_lock) { if (!io_poll_get_ownership(req)) return false; @@ -609,17 +616,23 @@ static bool io_poll_disarm(struct io_kiocb *req) } int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) - __must_hold(&ctx->completion_lock) { struct io_kiocb *req; + u32 index; + spinlock_t *lock; if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_ANY)) req = io_poll_file_find(ctx, cd); else req = io_poll_find(ctx, false, cd); - if (!req) + if (!req) { return -ENOENT; + } else { + index = hash_long(req->cqe.user_data, ctx->cancel_hash_bits); + lock = &ctx->cancel_hash[index].lock; + } io_poll_cancel_req(req); + spin_unlock(lock); return 0; } @@ -713,18 +726,23 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) struct io_poll_update *poll_update = io_kiocb_to_cmd(req); struct io_cancel_data cd = { .data = poll_update->old_user_data, }; struct io_ring_ctx *ctx = req->ctx; + u32 index = hash_long(cd.data, ctx->cancel_hash_bits); + spinlock_t *lock = &ctx->cancel_hash[index].lock; struct io_kiocb *preq; int ret2, ret = 0; bool locked; - spin_lock(&ctx->completion_lock); preq = io_poll_find(ctx, true, &cd); - if (!preq || !io_poll_disarm(preq)) { - spin_unlock(&ctx->completion_lock); - ret = preq ? -EALREADY : -ENOENT; + if (!preq) { + ret = -ENOENT; + goto out; + } + ret2 = io_poll_disarm(preq); + spin_unlock(lock); + if (!ret2) { + ret = -EALREADY; goto out; } - spin_unlock(&ctx->completion_lock); if (poll_update->update_events || poll_update->update_user_data) { /* only mask one event flags, keep behavior flags */ From patchwork Thu Jun 16 09:22:03 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883639 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0E103C43334 for ; Thu, 16 Jun 2022 09:22:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376608AbiFPJWy (ORCPT ); Thu, 16 Jun 2022 05:22:54 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47520 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376598AbiFPJWx (ORCPT ); Thu, 16 Jun 2022 05:22:53 -0400 Received: from mail-wr1-x42f.google.com (mail-wr1-x42f.google.com [IPv6:2a00:1450:4864:20::42f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 5C67212AA8 for ; Thu, 16 Jun 2022 02:22:51 -0700 (PDT) Received: by mail-wr1-x42f.google.com with SMTP id v14so1047874wra.5 for ; Thu, 16 Jun 2022 02:22:51 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=whmbV8SR6MxGfqcouaajDpb1UgDnXjtmeXGDkViNhtk=; b=hEhHNNPyPcojnH8NSTzWJtjEbPkBmJeG8eLtS51yz2etRQKNmS0zfOUPn/NJdM/Mtt eyuC561y9aA642aCcOv+TFW9NFHzs+2tckN+2REjcHo8RWXKSU46WkN0KWZlzT2KpbLG gtWtw4Aff31PZDc3JJFk9oeZdAYUhpyHa3BwpvS9JvkSky7gzKaeq5cLhnnJ54zVahCN SF2J+cQIbsdQiMpybnZ8VfmlYPJoLTOPyhfIYLUu06RjHfONRsEuLUlD5z2uOVh036Bj cKaYPt8ULC1G5/+YYPzU2sxh7Vjc5tBGO/93GFAGBGYP0KskHiUMTK8amg7tLbV6IlkQ JxkA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=whmbV8SR6MxGfqcouaajDpb1UgDnXjtmeXGDkViNhtk=; b=7jJGCiifLSYV9i+XiIoMmb+WDorlHLCjxrnDuJR6s8XdHOfQoVQLGjXi+SvM66vMP+ IcNN0FDUCEeefCH3sGy2hEZvIBmhilKxt+DRViyFabUR4ypp0SuVWwiA9wtaBU069tVD 11wFWS36dBj+p3FF5otydrNNzi0aIac9gEFgj+hySznENOABNsr7PP4nAI7q6IC6N7sV A6mqECpEirfnvrkGSC1/PATKcsX1bBFdWD1cab7ZCsnMCuBy1m25ZJ+ckB/tGwBx97zr trZziBIS2ne8CGWQh1UkrzT9K/pJFYDddAd4AT3JEsya6jruI23LVvRcsYWsedHnnZ+/ sltg== X-Gm-Message-State: AJIora8+Lul0vjb/7BdmtLXgn5YRL34CW0G/CloOT8npdGRSEmANicW8 QWqsvch+Tnwoa/BZHkrBqKf7QfsKIRVUKQ== X-Google-Smtp-Source: AGRyM1voXqs0XuKkli5bCNjrGzgT0qydjVlg9okjnvtnIb9QI5itkkKNlHlxsNzcwt3iYUyRGFCarg== X-Received: by 2002:a5d:4a88:0:b0:214:1e17:9993 with SMTP id o8-20020a5d4a88000000b002141e179993mr3610401wrq.608.1655371370561; Thu, 16 Jun 2022 02:22:50 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.49 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:50 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 07/16] io_uring: pass poll_find lock back Date: Thu, 16 Jun 2022 10:22:03 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Instead of using implicit knowledge of what is locked or not after io_poll_find() and co returns, pass back a pointer to the locked bucket if any. If set the user must to unlock the spinlock. Signed-off-by: Pavel Begunkov --- io_uring/poll.c | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/io_uring/poll.c b/io_uring/poll.c index 7f6b16f687b0..7fc4aafcca95 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -559,12 +559,15 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, } static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, - struct io_cancel_data *cd) + struct io_cancel_data *cd, + struct io_hash_bucket **out_bucket) { struct io_kiocb *req; u32 index = hash_long(cd->data, ctx->cancel_hash_bits); struct io_hash_bucket *hb = &ctx->cancel_hash[index]; + *out_bucket = NULL; + spin_lock(&hb->lock); hlist_for_each_entry(req, &hb->list, hash_node) { if (cd->data != req->cqe.user_data) @@ -576,6 +579,7 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, continue; req->work.cancel_seq = cd->seq; } + *out_bucket = hb; return req; } spin_unlock(&hb->lock); @@ -583,11 +587,14 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, } static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, - struct io_cancel_data *cd) + struct io_cancel_data *cd, + struct io_hash_bucket **out_bucket) { struct io_kiocb *req; int i; + *out_bucket = NULL; + for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { struct io_hash_bucket *hb = &ctx->cancel_hash[i]; @@ -599,6 +606,7 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, if (cd->seq == req->work.cancel_seq) continue; req->work.cancel_seq = cd->seq; + *out_bucket = hb; return req; } spin_unlock(&hb->lock); @@ -617,23 +625,19 @@ static bool io_poll_disarm(struct io_kiocb *req) int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) { + struct io_hash_bucket *bucket; struct io_kiocb *req; - u32 index; - spinlock_t *lock; if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_ANY)) - req = io_poll_file_find(ctx, cd); + req = io_poll_file_find(ctx, cd, &bucket); else - req = io_poll_find(ctx, false, cd); - if (!req) { - return -ENOENT; - } else { - index = hash_long(req->cqe.user_data, ctx->cancel_hash_bits); - lock = &ctx->cancel_hash[index].lock; - } - io_poll_cancel_req(req); - spin_unlock(lock); - return 0; + req = io_poll_find(ctx, false, cd, &bucket); + + if (req) + io_poll_cancel_req(req); + if (bucket) + spin_unlock(&bucket->lock); + return req ? 0 : -ENOENT; } static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, @@ -726,19 +730,21 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) struct io_poll_update *poll_update = io_kiocb_to_cmd(req); struct io_cancel_data cd = { .data = poll_update->old_user_data, }; struct io_ring_ctx *ctx = req->ctx; - u32 index = hash_long(cd.data, ctx->cancel_hash_bits); - spinlock_t *lock = &ctx->cancel_hash[index].lock; + struct io_hash_bucket *bucket; struct io_kiocb *preq; int ret2, ret = 0; bool locked; - preq = io_poll_find(ctx, true, &cd); + preq = io_poll_find(ctx, true, &cd, &bucket); + if (preq) + ret2 = io_poll_disarm(preq); + if (bucket) + spin_unlock(&bucket->lock); + if (!preq) { ret = -ENOENT; goto out; } - ret2 = io_poll_disarm(preq); - spin_unlock(lock); if (!ret2) { ret = -EALREADY; goto out; From patchwork Thu Jun 16 09:22:04 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883641 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id E8FC7CCA47A for ; Thu, 16 Jun 2022 09:22:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376598AbiFPJW4 (ORCPT ); Thu, 16 Jun 2022 05:22:56 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47566 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376331AbiFPJWy (ORCPT ); Thu, 16 Jun 2022 05:22:54 -0400 Received: from mail-wr1-x431.google.com (mail-wr1-x431.google.com [IPv6:2a00:1450:4864:20::431]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 5CCCC17057 for ; Thu, 16 Jun 2022 02:22:53 -0700 (PDT) Received: by mail-wr1-x431.google.com with SMTP id o8so1054025wro.3 for ; Thu, 16 Jun 2022 02:22:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=6Mn1I3qJJuXoCLtaTkB3ey7V/NeeqbASVSbKJNj/gRM=; b=Z7+mUXTNB35OQnpsqRPaI5dS+t/mgptcxWtGmZhp4NgBLVvWddQfCUU4oSRQuVuNJD I9GqF2pzLSbTwTUtMj3z35bT9s8zbgm8Sw3ETvjSMn+YGcI3oTFt+If0arwQ3i6G/dEa UpVDKkTrskuZACf0S8JtvIU9iU3Et+zmpmscPiBWSSuvnOjuOETDrxmVAtpRHDAaQu6a 1pe2r0QF8WiVWNuxgE2eU/ClKQGXx6b7NbCU5XQn6GCSb9HWq7bM5f7Vp7HBE0uwI5YF QyGjomPsLfy32sneKlpwoj1cBiFaRlrT6RMW1KNc5Z1PL9Wp9caEmveSi3LL7iVVRvWb KCUg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=6Mn1I3qJJuXoCLtaTkB3ey7V/NeeqbASVSbKJNj/gRM=; b=mSVp4oYwmKG6I3jWLIkHAEKQWd2+BTyBww0DRALpM8oqOIEbxAP/b1r8+Rh/UAUR/7 b3D5MIsCxCuzqjDpwu9ZUbHK1ujfJ/lOx6fR1Lk5TV6Hx5PyfOl4N85qQBXEZJm851Tz behQrjcs6WoxATfivbj9Kd49hjZQ3aQ0B9KePX3x2IsXYuWBoF65HaRpmXHw47fcN+8u IcCsK08C4ROOJLxZQ9BTRi6BY69cudG5UXnAOspkSYeghUl2fuvGZh51xlRou5H5Z6pn JcCqWALc0Is7kjzUQKKNnCrpveUxDJ942PhrTFBWE5415ly2k1LvwSQOgU+DJatLXtGw ElZA== X-Gm-Message-State: AJIora+x9O+7NKNzP2wXdBgDlejykPGgDkiSm0XVW+0wrr+MKAuO6OMz SNk6x72WBaEYsvzs9BtPmsRbNdUMTSFs6Q== X-Google-Smtp-Source: AGRyM1t3o6SPPi10wZfI+Y5jjR6hpPLZRxLt8ZGyf68AadoyLyrVSoLnVhVPLpELcpovPby/NHgj2A== X-Received: by 2002:a5d:4108:0:b0:213:b585:66c7 with SMTP id l8-20020a5d4108000000b00213b58566c7mr3644926wrp.335.1655371371587; Thu, 16 Jun 2022 02:22:51 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.50 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:51 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 08/16] io_uring: clean up io_try_cancel Date: Thu, 16 Jun 2022 10:22:04 +0100 Message-Id: <48cf5417b43a8386c6c364dba1ad9b4c7382d158.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Get rid of an unnecessary extra goto in io_try_cancel() and simplify the function. Signed-off-by: Pavel Begunkov --- io_uring/cancel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 6f2888388a40..a253e2ad22eb 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -95,12 +95,12 @@ int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd) ret = io_poll_cancel(ctx, cd); if (ret != -ENOENT) - goto out; + return ret; + spin_lock(&ctx->completion_lock); if (!(cd->flags & IORING_ASYNC_CANCEL_FD)) ret = io_timeout_cancel(ctx, cd); spin_unlock(&ctx->completion_lock); -out: return ret; } From patchwork Thu Jun 16 09:22:05 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883642 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4E60FCCA47D for ; Thu, 16 Jun 2022 09:22:57 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376331AbiFPJW4 (ORCPT ); Thu, 16 Jun 2022 05:22:56 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47568 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376481AbiFPJWy (ORCPT ); Thu, 16 Jun 2022 05:22:54 -0400 Received: from mail-wr1-x42b.google.com (mail-wr1-x42b.google.com [IPv6:2a00:1450:4864:20::42b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 7749B11174 for ; Thu, 16 Jun 2022 02:22:53 -0700 (PDT) Received: by mail-wr1-x42b.google.com with SMTP id s1so1035718wra.9 for ; Thu, 16 Jun 2022 02:22:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=fb6MpG+TUR+lfc/l0HsFy1uGwvkTzMYMEuxuESOhYE4=; b=HrvVHvyie15H815z2sNl4TYkIdl/No+fxC/1/XFc3hlf6Yo/sNS0K7guQ+3JtGQ4ly fo7PT5J8o1BUdR+SC2LMvBc5N4+1LvdVGHHTh+Qkxj0iEXqKXXTIYbePhyUBCk/K8q5R /WN+k19jJht4AYhzrzLexgnCjBaRK6A8CK3ruP7xkHRc251tOMHsZetd2iAyK7biVYb5 jVdUlZXNEllhZlhLfS5rEtAjvaqi6FBhY3YgE4k4xrMCwChJg82fGoQZtY/JvNXaK1Rm V1l1NVm/IQNRfiJTUUPOJyVK5SRi1BSlSpaMvcO9KR2cdJ4tXV395W/lwIS2FhxZ3kUK FOMg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=fb6MpG+TUR+lfc/l0HsFy1uGwvkTzMYMEuxuESOhYE4=; b=PwaRnlnCb/ZFdn2JITRC4PmGU9J5TkRdUEFb3ndWqdHuxJ+sASoDD2M9mB0tcbwMv8 lwJSYYPeFkLksNmr/s9zb6pMi6VjMnIctyo6yS1EoZ/ODkvZ3hckuj5PK5R+iyq70PZj b6DzD6ALCPxNT68T4SW2PXBEJYOMHhZNXzsDAhftrfccnZ5EjhMDv5AiV2xSS7rqP2AJ IbbljO1Y9LTV3V+rqhaVx/X87U98Ok/J7Wwtn+HKFtkUQAAUsyjKNDuQIAV1b1ESoUT5 1dgYLgjUhke24Pw5hUgJqZi6yPczEJbwLMV+1HnV+bOFwRHRxQyvOTqPLPrabwVzQONZ mZnw== X-Gm-Message-State: AJIora8SIOXZdWvlekBBOiHf7NJVzPcrNpzxc2lrOjPr99UoDn5EkLAu Chf9HGN2f0xhTkJ26NdmNOEd/iE+1VjKEg== X-Google-Smtp-Source: AGRyM1uzuD58Gvw8ldCGsMjEj9bif5uM5n1mI+w4DxkZrCSYFjIOptjgJbeQYnZve+HKVZa2FPRdFw== X-Received: by 2002:a5d:6d0d:0:b0:218:45b1:ef1f with SMTP id e13-20020a5d6d0d000000b0021845b1ef1fmr3784547wrq.558.1655371372744; Thu, 16 Jun 2022 02:22:52 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.51 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:52 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 09/16] io_uring: limit the number of cancellation buckets Date: Thu, 16 Jun 2022 10:22:05 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Don't allocate to many hash/cancellation buckets, there might be too many, clamp it to 8 bits, or 256 * 64B = 16KB. We don't usually have too many requests, and 256 buckets should be enough, especially since we do hash search only in the cancellation path. Signed-off-by: Pavel Begunkov --- io_uring/io_uring.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index d0242e5c8d0a..97113c71e881 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -254,12 +254,12 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) /* * Use 5 bits less than the max cq entries, that should give us around - * 32 entries per hash list if totally full and uniformly spread. + * 32 entries per hash list if totally full and uniformly spread, but + * don't keep too many buckets to not overconsume memory. */ - hash_bits = ilog2(p->cq_entries); - hash_bits -= 5; - if (hash_bits <= 0) - hash_bits = 1; + hash_bits = ilog2(p->cq_entries) - 5; + hash_bits = clamp(hash_bits, 1, 8); + ctx->cancel_hash_bits = hash_bits; ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct io_hash_bucket), From patchwork Thu Jun 16 09:22:06 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883643 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id C0EB8CCA480 for ; Thu, 16 Jun 2022 09:22:57 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376481AbiFPJW5 (ORCPT ); Thu, 16 Jun 2022 05:22:57 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47604 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376618AbiFPJWz (ORCPT ); Thu, 16 Jun 2022 05:22:55 -0400 Received: from mail-wr1-x42b.google.com (mail-wr1-x42b.google.com [IPv6:2a00:1450:4864:20::42b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 7C6D817049 for ; Thu, 16 Jun 2022 02:22:54 -0700 (PDT) Received: by mail-wr1-x42b.google.com with SMTP id o8so1053920wro.3 for ; Thu, 16 Jun 2022 02:22:54 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=fI7Bbf1p3N5077YqPdGx1gf2DBASuHPaoVFSYzfe9G4=; b=gGOzo5KQQ2bO3MPeuhUtkTDiLYqLzRWPzGiZitZ9rCtvqAPR1uoLxDWjasbKPg1WkN qvIFm5NJWiGp5/kdLpGaPq+sF2DP/YDT6FV0+f0GSNWilKx9kNbJT/HPj8awFlyJFCwt jh6aDqACk1WSMguSZ9V+33FTkhwym620pkfTjj80o/9GyUvyOsCPYO6zYGVDnZ9ENJBG 0Xkzi13GqMV15eemw0bNknqHHCq0HuhD233OTHEG72Z85mBo2HSOvrqSIiOHQua8TujA dDhjX2UQ9uOhgD0YhPd3JOH/Rg6k0xu3sAu1gwAWDuXOb1PmeqR0wJzf2/2jn/pjLCZb dwvw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=fI7Bbf1p3N5077YqPdGx1gf2DBASuHPaoVFSYzfe9G4=; b=ukw7zO6vbrIXn5Ge88KdPBtOgo4VeMnzZMzcdKDDX+DAb2YxWLUF7hv1EFeWrDXVc8 ucIay+Et0Rka5mcT/k+lGxy9dnpghFVi41RwoxeLLi4Vsulrx05g6mBQRU3qyFRmwUgJ OBCdG8ncd1iuueTi1rKo5T8CIHZiXH1eNaH/ms+ZHWnVWHTEGDv3RIV6/PtOoNtFr1Ha az3zOVjHhNG/sN2Duzyzx2ZWDibWKJ8T4N7wpcIbSq6fftMhJ0vljw45pVB68gGpeyMO seYiharb7oYEKWoZk9FqrHnEN4bBVFy7uAYgRpsFzRuoWH+V1GcVhKe9OOuuSCQLn81w x1Ag== X-Gm-Message-State: AJIora8jPAMvhBPairRbddikqyvwa4dWkJT+mXQydYdrVmra8HKszF4D BlzbOPNXYR4uzvJ1kqu4moc2ySYlBfbEiA== X-Google-Smtp-Source: AGRyM1tCH6WKkwfbkWJWWCjNyUQTPbG6iiX8qesDp16qFpDzru8B2wT0stBWUeqhrI3b8jL1fppZlg== X-Received: by 2002:a5d:6484:0:b0:219:eb95:3502 with SMTP id o4-20020a5d6484000000b00219eb953502mr3708525wri.692.1655371373764; Thu, 16 Jun 2022 02:22:53 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.52 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:53 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 10/16] io_uring: clean up io_ring_ctx_alloc Date: Thu, 16 Jun 2022 10:22:06 +0100 Message-Id: <993926ed0d614ba9a76b2a85bebae2babcb13983.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Add a variable for the number of hash buckets in io_ring_ctx_alloc(), makes it more readable. Signed-off-by: Pavel Begunkov --- io_uring/io_uring.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 97113c71e881..b2bd71cb2be6 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -244,6 +244,8 @@ static __cold void io_fallback_req_func(struct work_struct *work) static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) { struct io_ring_ctx *ctx; + unsigned hash_buckets; + size_t hash_size; int hash_bits; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); @@ -259,15 +261,15 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) */ hash_bits = ilog2(p->cq_entries) - 5; hash_bits = clamp(hash_bits, 1, 8); + hash_buckets = 1U << hash_bits; + hash_size = hash_buckets * sizeof(struct io_hash_bucket); ctx->cancel_hash_bits = hash_bits; - ctx->cancel_hash = - kmalloc((1U << hash_bits) * sizeof(struct io_hash_bucket), - GFP_KERNEL); + ctx->cancel_hash = kmalloc(hash_size, GFP_KERNEL); if (!ctx->cancel_hash) goto err; - init_hash_table(ctx->cancel_hash, 1U << hash_bits); + init_hash_table(ctx->cancel_hash, hash_buckets); ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL); if (!ctx->dummy_ubuf) From patchwork Thu Jun 16 09:22:07 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883644 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3F86BC433EF for ; Thu, 16 Jun 2022 09:22:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376618AbiFPJW5 (ORCPT ); Thu, 16 Jun 2022 05:22:57 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47642 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376658AbiFPJW4 (ORCPT ); Thu, 16 Jun 2022 05:22:56 -0400 Received: from mail-wm1-x332.google.com (mail-wm1-x332.google.com [IPv6:2a00:1450:4864:20::332]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A3CF218B22 for ; Thu, 16 Jun 2022 02:22:55 -0700 (PDT) Received: by mail-wm1-x332.google.com with SMTP id a10so422660wmj.5 for ; Thu, 16 Jun 2022 02:22:55 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=ROudprvWUcWfvZ4cZzMiGRRFxKziH9KPUlRbWRK/uOQ=; b=k5zKmDzuu1oD7DtPkEmYzle27J8WXkk4X8Q/FZKoXLTsKw+FmC9+62jQ/Z47ZOzOzJ fbWgF9g6NNyi2kc7FygAxwseyuXzymGnBR1KFo7ZEOsEkYO4vx82RMXiL/g0ljGpzb6x wKw1E2uZJEcamOlKdumzAj3FrbatOJDuz2ln5gC9ILAXjFTn9tgJo8uJZQVlWDEc6Y0r hYN4p4MNzKYEgn8wZe6zfvS43GRGA1HuPFiMc5ZSVKHS2N+ggNd4j7baiQwD7NA8Q8Vt Thi+SdThj0puZqtwUsn1+bArS/AXXmGe2U6oJEUg+KCFJ53RPWstYax5EYbQVWWcKr+I MDeA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=ROudprvWUcWfvZ4cZzMiGRRFxKziH9KPUlRbWRK/uOQ=; b=3HHpixPIcPOUC4a6v+/9LehrOonuKuPwghTj4acnKc5hKJ3IergvE6bXcUbLLtOy6k Y0gCN5wRHNI19hag+OnQ6QvA/YU1auDmaaH7BjFQHQKwZqxQQM+P4u/6wFKJSDoVd2i5 OSNDX6BsZ2KskJ1V1kCZEQnwt9P7XV3WUFU0dglGxqG+D0mBllIoZeuUKcierJ7qFN0I IS8j/f3Sk5qSKR4osigOA7d/FrstGd6rz6A5DP4CpMMb1SMpu1rwjB/kUuaGaEiHPj3f hQ2B7liGCd6IJlLuh2Hir3doKyXim+NLTV8alPNPBxlTXklWz82F1Tv43mMtIjTQ4r+b Zzhw== X-Gm-Message-State: AJIora9KiN5FY13tRChqCiJqMSsfSEPYdfHnPucVndRmvBOVicNnrM2a D+H3DE0IQkhNFqivHzyGp2766CYSQ4priw== X-Google-Smtp-Source: AGRyM1vOn+u+U4Pz/ZWUx54MJ1ZLVXc8b4NB3NumBNUkf5inJnC3LZlx0B4n6B4yo6LNajeYEj1Y8A== X-Received: by 2002:a05:600c:3d8e:b0:39c:573b:3079 with SMTP id bi14-20020a05600c3d8e00b0039c573b3079mr3957780wmb.131.1655371374881; Thu, 16 Jun 2022 02:22:54 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.53 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:54 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 11/16] io_uring: use state completion infra for poll reqs Date: Thu, 16 Jun 2022 10:22:07 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Use io_req_task_complete() for poll request completions, so it can utilise state completions and save lots of unnecessary locking. Signed-off-by: Pavel Begunkov --- io_uring/poll.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/io_uring/poll.c b/io_uring/poll.c index 7fc4aafcca95..c4ce98504986 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -234,12 +234,8 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) io_poll_remove_entries(req); io_poll_req_delete(req, ctx); - spin_lock(&ctx->completion_lock); - req->cqe.flags = 0; - __io_req_complete_post(req); - io_commit_cqring(ctx); - spin_unlock(&ctx->completion_lock); - io_cqring_ev_posted(ctx); + io_req_set_res(req, req->cqe.res, 0); + io_req_task_complete(req, locked); } static void io_apoll_task_func(struct io_kiocb *req, bool *locked) From patchwork Thu Jun 16 09:22:08 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883645 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 1B46DC43334 for ; Thu, 16 Jun 2022 09:22:59 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376656AbiFPJW6 (ORCPT ); Thu, 16 Jun 2022 05:22:58 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47688 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376655AbiFPJW5 (ORCPT ); Thu, 16 Jun 2022 05:22:57 -0400 Received: from mail-wr1-x436.google.com (mail-wr1-x436.google.com [IPv6:2a00:1450:4864:20::436]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E0FCC17057 for ; Thu, 16 Jun 2022 02:22:56 -0700 (PDT) Received: by mail-wr1-x436.google.com with SMTP id m24so1030919wrb.10 for ; Thu, 16 Jun 2022 02:22:56 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=ANZk26/u+omi5h8PfmD2ift8n+Vb8egwsV2O9IpMZAk=; b=pZOIM2JWCad4wO8c9t6qYHZ7p/NBcvSxTdksULpHBNvTyWJ09TLLrLgER2TkjzkJR+ e6FGpGpjk7OIH/8RgshqENVouWe7YDAZUyfBRmx6APegNen59L1tM/bOEjgWTiSJ8coc nK9RIBflJoPqsGweq0ZkDLmuL/sRug7h/g/mX2kGMwSrPor6gaXyeIoTVW5QGkx1CFBL 5SFbh2hSze5wW0JqX+c6DeNUt2TAgqaV0svz0k2CWBO3dGgnRSIfTCg+IRN3EFdqJDqg RqBLWF2VRY2+VRw7+sJPifM+/ZXQl/XwTjohIiKP9g4yfPOUsuSmio9CX0cXUhvfszQj qzOA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=ANZk26/u+omi5h8PfmD2ift8n+Vb8egwsV2O9IpMZAk=; b=ZyUSXW7SgSyyDRH0G0U3vQ1NzrQuUfZ6YSEIw51g3pn6bf5FUlSHXpaamiKoh+/lYm tePFnDI8HbPSfyUUuLfgE8xuUvEZnDN3TUCPgf8F//8aNXQSL+21FXhCl/xfoIa1aeyH +2D+1VIwGKD5kxMG514Bs0uBI7JjonE+2E+UwyuGCj3TOhOAMFsdtSBt3mbZMgfi8coj 7v7zkgLa3E75+OlJr7hTroTZhATSTrLkjcSkeT7IM5d+96KflgCVp0fMUnFHKqljN5LL +QraYCeuJO2BEt01wQww1RGy+LTpCtANgGZCFpdgrSiNOMCa8wKMeiuSQdrZ6Kwq7Wv5 bR3A== X-Gm-Message-State: AJIora8sljmhyH5aGmQkv+O51j5WU5+IdUV9YXMpd025cmrjTtXtitbz LOgGBNhwoK/T+TX+a1RoBh7MTYuHkgGuEw== X-Google-Smtp-Source: AGRyM1uCj61qYBxSs2AhG+uKDRpRfZi0zyeTesNRyiVXmAMXseTVhRg0dGXq4mG2rXpY/rtE9MuZVg== X-Received: by 2002:a05:6000:186e:b0:218:5f5d:9c55 with SMTP id d14-20020a056000186e00b002185f5d9c55mr3563392wri.128.1655371375989; Thu, 16 Jun 2022 02:22:55 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.55 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:55 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 12/16] io_uring: add IORING_SETUP_SINGLE_ISSUER Date: Thu, 16 Jun 2022 10:22:08 +0100 Message-Id: <4bcc41ee467fdf04c8aab8baf6ce3ba21858c3d4.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Add a new IORING_SETUP_SINGLE_ISSUER flag and the userspace visible part of it, i.e. put limitations of submitters. Also, don't allow it together with IOPOLL as we're not going to put it to good use. Signed-off-by: Pavel Begunkov --- include/uapi/linux/io_uring.h | 5 ++++- io_uring/io_uring.c | 7 +++++-- io_uring/io_uring_types.h | 1 + io_uring/tctx.c | 27 ++++++++++++++++++++++++--- io_uring/tctx.h | 4 ++-- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 1d176f935f5d..8715f0942ec2 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -137,9 +137,12 @@ enum { * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN. */ #define IORING_SETUP_TASKRUN_FLAG (1U << 9) - #define IORING_SETUP_SQE128 (1U << 10) /* SQEs are 128 byte */ #define IORING_SETUP_CQE32 (1U << 11) /* CQEs are 32 byte */ +/* + * Only one task is allowed to submit requests + */ +#define IORING_SETUP_SINGLE_ISSUER (1U << 12) enum io_uring_op { IORING_OP_NOP, diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b2bd71cb2be6..1308a1f0c349 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2469,6 +2469,8 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_destroy_buffers(ctx); if (ctx->sq_creds) put_cred(ctx->sq_creds); + if (ctx->submitter_task) + put_task_struct(ctx->submitter_task); /* there are no registered resources left, nobody uses it */ if (ctx->rsrc_node) @@ -3201,7 +3203,7 @@ static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) if (fd < 0) return fd; - ret = io_uring_add_tctx_node(ctx); + ret = __io_uring_add_tctx_node(ctx, false); if (ret) { put_unused_fd(fd); return ret; @@ -3421,7 +3423,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ | IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | - IORING_SETUP_SQE128 | IORING_SETUP_CQE32)) + IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | + IORING_SETUP_SINGLE_ISSUER)) return -EINVAL; return io_uring_create(entries, &p, params); diff --git a/io_uring/io_uring_types.h b/io_uring/io_uring_types.h index 1f8db2dd7af7..8b00243abf65 100644 --- a/io_uring/io_uring_types.h +++ b/io_uring/io_uring_types.h @@ -243,6 +243,7 @@ struct io_ring_ctx { /* Keep this last, we don't need it for the fast path */ struct io_restriction restrictions; + struct task_struct *submitter_task; /* slow path rsrc auxilary data, used by update/register */ struct io_rsrc_node *rsrc_backup_node; diff --git a/io_uring/tctx.c b/io_uring/tctx.c index 6adf659687f8..012be261dc50 100644 --- a/io_uring/tctx.c +++ b/io_uring/tctx.c @@ -81,12 +81,32 @@ __cold int io_uring_alloc_task_context(struct task_struct *task, return 0; } -int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) +static int io_register_submitter(struct io_ring_ctx *ctx) +{ + int ret = 0; + + mutex_lock(&ctx->uring_lock); + if (!ctx->submitter_task) + ctx->submitter_task = get_task_struct(current); + else if (ctx->submitter_task != current) + ret = -EEXIST; + mutex_unlock(&ctx->uring_lock); + + return ret; +} + +int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; int ret; + if ((ctx->flags & IORING_SETUP_SINGLE_ISSUER) && submitter) { + ret = io_register_submitter(ctx); + if (ret) + return ret; + } + if (unlikely(!tctx)) { ret = io_uring_alloc_task_context(current, ctx); if (unlikely(ret)) @@ -120,7 +140,8 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) list_add(&node->ctx_node, &ctx->tctx_list); mutex_unlock(&ctx->uring_lock); } - tctx->last = ctx; + if (submitter) + tctx->last = ctx; return 0; } @@ -228,7 +249,7 @@ int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg, return -EINVAL; mutex_unlock(&ctx->uring_lock); - ret = io_uring_add_tctx_node(ctx); + ret = __io_uring_add_tctx_node(ctx, false); mutex_lock(&ctx->uring_lock); if (ret) return ret; diff --git a/io_uring/tctx.h b/io_uring/tctx.h index 7684713e950f..dde82ce4d8e2 100644 --- a/io_uring/tctx.h +++ b/io_uring/tctx.h @@ -34,7 +34,7 @@ struct io_tctx_node { int io_uring_alloc_task_context(struct task_struct *task, struct io_ring_ctx *ctx); void io_uring_del_tctx_node(unsigned long index); -int __io_uring_add_tctx_node(struct io_ring_ctx *ctx); +int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter); void io_uring_clean_tctx(struct io_uring_task *tctx); void io_uring_unreg_ringfd(void); @@ -52,5 +52,5 @@ static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) if (likely(tctx && tctx->last == ctx)) return 0; - return __io_uring_add_tctx_node(ctx); + return __io_uring_add_tctx_node(ctx, true); } From patchwork Thu Jun 16 09:22:09 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883646 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id EDA5DC43334 for ; Thu, 16 Jun 2022 09:23:03 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376668AbiFPJXC (ORCPT ); Thu, 16 Jun 2022 05:23:02 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47760 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376655AbiFPJXA (ORCPT ); Thu, 16 Jun 2022 05:23:00 -0400 Received: from mail-wm1-x32a.google.com (mail-wm1-x32a.google.com [IPv6:2a00:1450:4864:20::32a]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E9ACC193E3 for ; Thu, 16 Jun 2022 02:22:58 -0700 (PDT) Received: by mail-wm1-x32a.google.com with SMTP id l126-20020a1c2584000000b0039c1a10507fso534752wml.1 for ; Thu, 16 Jun 2022 02:22:58 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=5V/K45g9wpqoJNn9BXBkT927aYIh71OeYAeKJeObVFk=; b=RlquVMAPlEOISo16lDxOP4xOacXdN+leilPiO/Lk4KZix4LB9WikcmcRs/X7DDvddg L3ja85OVVziR9M0OuHFe0CI+o0MCbZNCU/wF5EMFZe4sfTZkmRIA1ayF6bHrITCYwsHH fA75dfCvtJEnRNky32z4JjZErV3uXwJ2ciXhlNuwbK/tiaVBkrzIX62vvJ3FPRaTd8sv sgDWVs6din4XrMl80fFsuDrqjrddAVIujkaMs1+3wDeFl1dWcuj5MjTUjzWGkN1eN6Gn Ap8zrv0Ii8/0odF5+ozUVwDR/tz4Zjfam3gTjawIPuslflNMEtI9fhAPMQBLDlDEYOpq GY3A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=5V/K45g9wpqoJNn9BXBkT927aYIh71OeYAeKJeObVFk=; b=iv0lSfPQerfB52nFOpGskq6+Tu48iTwDf6TRhKjbuKLzUxD/0a8WfhCNJHP3AXo01T 34KfL1nUSs0J4totf80aEJZpn7T+rSGNDJBMABzFXcIYUhFPMi3q+C3RPgQW6Ju2pRy2 L3EOc9IWwuM4cs1vRN4arYE8isN9YDVhAl4EMma0lAZvxfppmnXxPKdK2DvHZhE0+3uO kNvbBjE2zGwGKh0SN5W1UUDBlUNVHbt3+vmalfITp8QmFI5SfuUzjtk3NUFvuuPeu10r dEfNm7O/EWhHo7YCwkoSwHyzskgQZ7jQmwl2QMIjmV3Xxm/pYLl102hmiMtLV9fkz8TD LHLA== X-Gm-Message-State: AJIora9VPtNNHto6HcjOQAe35tSVs5GDN9LgkfojVlCvbht5LoKXhWWH EHBaQRdU7Z6AgCcySDgqmbvPXvvL1MAe6A== X-Google-Smtp-Source: AGRyM1ugt6JI3aGYVI9UPAlUPBjFc0pPKRZuyYv+a4tfnunKMCA+jNTakE9eRUOy/oOBOAf3UgOKbQ== X-Received: by 2002:a05:600c:3c8f:b0:39b:808c:b5cb with SMTP id bg15-20020a05600c3c8f00b0039b808cb5cbmr3918015wmb.11.1655371377244; Thu, 16 Jun 2022 02:22:57 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.56 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:56 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 13/16] io_uring: pass hash table into poll_find Date: Thu, 16 Jun 2022 10:22:09 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org In preparation for having multiple cancellation hash tables, pass a table pointer into io_poll_find() and other poll cancel functions. Signed-off-by: Pavel Begunkov --- io_uring/poll.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/io_uring/poll.c b/io_uring/poll.c index c4ce98504986..5cc03be365e3 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -556,11 +556,12 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, struct io_cancel_data *cd, + struct io_hash_bucket hash_table[], struct io_hash_bucket **out_bucket) { struct io_kiocb *req; u32 index = hash_long(cd->data, ctx->cancel_hash_bits); - struct io_hash_bucket *hb = &ctx->cancel_hash[index]; + struct io_hash_bucket *hb = &hash_table[index]; *out_bucket = NULL; @@ -584,6 +585,7 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, struct io_cancel_data *cd, + struct io_hash_bucket hash_table[], struct io_hash_bucket **out_bucket) { struct io_kiocb *req; @@ -592,7 +594,7 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, *out_bucket = NULL; for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { - struct io_hash_bucket *hb = &ctx->cancel_hash[i]; + struct io_hash_bucket *hb = &hash_table[i]; spin_lock(&hb->lock); hlist_for_each_entry(req, &hb->list, hash_node) { @@ -619,15 +621,16 @@ static bool io_poll_disarm(struct io_kiocb *req) return true; } -int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) +static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, + struct io_hash_bucket hash_table[]) { struct io_hash_bucket *bucket; struct io_kiocb *req; if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_ANY)) - req = io_poll_file_find(ctx, cd, &bucket); + req = io_poll_file_find(ctx, cd, ctx->cancel_hash, &bucket); else - req = io_poll_find(ctx, false, cd, &bucket); + req = io_poll_find(ctx, false, cd, ctx->cancel_hash, &bucket); if (req) io_poll_cancel_req(req); @@ -636,6 +639,11 @@ int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) return req ? 0 : -ENOENT; } +int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) +{ + return __io_poll_cancel(ctx, cd, ctx->cancel_hash); +} + static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, unsigned int flags) { @@ -731,7 +739,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) int ret2, ret = 0; bool locked; - preq = io_poll_find(ctx, true, &cd, &bucket); + preq = io_poll_find(ctx, true, &cd, ctx->cancel_hash, &bucket); if (preq) ret2 = io_poll_disarm(preq); if (bucket) From patchwork Thu Jun 16 09:22:10 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883647 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 82244C433EF for ; Thu, 16 Jun 2022 09:23:04 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376655AbiFPJXD (ORCPT ); Thu, 16 Jun 2022 05:23:03 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47806 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376658AbiFPJXB (ORCPT ); Thu, 16 Jun 2022 05:23:01 -0400 Received: from mail-wr1-x42b.google.com (mail-wr1-x42b.google.com [IPv6:2a00:1450:4864:20::42b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 15CC71C904 for ; Thu, 16 Jun 2022 02:23:00 -0700 (PDT) Received: by mail-wr1-x42b.google.com with SMTP id h5so1083354wrb.0 for ; Thu, 16 Jun 2022 02:23:00 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=t3CNGqUawoU0UR5moxpevr6kT0/FhpmPnzRqicn0Og0=; b=Th/AqXEwAmQj82MZDumFY+Rs18yvkTEQYdsG5XamRiDP9ZpJEgAYQmloPKBCpxhoX5 87eo2HhzUJcXQY3b4gdeNthgWynsSNE5mFjkbqAAKHSJHhrjo661q8jNj1QyryxPXI8c MF1cHdMa4LPHV+Qwqr+0ehBeephtpv5WvO9ogr4z5QSoSBiOqZpW+8Y97Ev+VhyZiA5O ZABvq53WPX8EmbqL0bM46j/VtUcfU51FA5R+IbWSYVL3E8P4yVSrCnwr/li0Mrqm4oPV PoEjjtIcfiMf0Cc/VhL1qzzv0/AYUiC1bv51JiUUnBQTq10qXq1JYSEgkk+VoQP7qdwy djWA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=t3CNGqUawoU0UR5moxpevr6kT0/FhpmPnzRqicn0Og0=; b=4tinSUT8Y5tKkagS5TS7qOYy5oxYr9xd+wbz+rjie1DVUatPrWe0DXHFj+H41Rj0pd ZxnqQ2LBUkyhsCyedZ1dnG6Rr2qAOXx1RESuMhP0x8ir6TwLia/n/zI/WPS/zXWuUoyY 0oW+Vr3CrUbDDObAQN/d2RkjYo58vHUVGdHIdKGEdvXYBEOb7FxIpDX/tW+eKoGU5X8F p9sXx/HqHqWdTS50W5iEN06YPeD5T5T8sM8ulbAY04c0ItaXhmOhKjGzkm4ItPrcPCNB hnj4lDv2mfqlxqxyRGbA7rhu4w9PrgKWBrFlZ51rGO61YLiI1zDXvdoeI0Rm3x1eaSLp xPcQ== X-Gm-Message-State: AJIora8coqLj3NP+FUDvv6yA7JSk64iDktxyu+aSRB1cvADgwQhwm1Hx 5cu9FFuX6neW9d6f7btkIzUrzEh26Zb3tw== X-Google-Smtp-Source: AGRyM1tSa7Fbd2VCK3ezCY6ZnrmNUsL3ksBBb5Y3Ucwk3OE0WKV9Jj5weXgRiamfq40L5OWaAUnuGg== X-Received: by 2002:a05:6000:15ca:b0:218:48a7:a45f with SMTP id y10-20020a05600015ca00b0021848a7a45fmr3730066wry.591.1655371378278; Thu, 16 Jun 2022 02:22:58 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.57 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:57 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 14/16] io_uring: introduce a struct for hash table Date: Thu, 16 Jun 2022 10:22:10 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Instead of passing around a pointer to hash buckets, add a bit of type safety and wrap it into a structure. Signed-off-by: Pavel Begunkov --- io_uring/cancel.c | 6 +++--- io_uring/cancel.h | 7 +------ io_uring/fdinfo.c | 4 ++-- io_uring/io_uring.c | 29 ++++++++++++++++------------ io_uring/io_uring_types.h | 13 +++++++++++-- io_uring/poll.c | 40 +++++++++++++++++++++------------------ 6 files changed, 56 insertions(+), 43 deletions(-) diff --git a/io_uring/cancel.c b/io_uring/cancel.c index a253e2ad22eb..f28f0a7d1272 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -193,12 +193,12 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) return IOU_OK; } -void init_hash_table(struct io_hash_bucket *hash_table, unsigned size) +void init_hash_table(struct io_hash_table *table, unsigned size) { unsigned int i; for (i = 0; i < size; i++) { - spin_lock_init(&hash_table[i].lock); - INIT_HLIST_HEAD(&hash_table[i].list); + spin_lock_init(&table->hbs[i].lock); + INIT_HLIST_HEAD(&table->hbs[i].list); } } diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 556a7dcf160e..fd4cb1a2595d 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -4,9 +4,4 @@ int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags); int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd); -void init_hash_table(struct io_hash_bucket *hash_table, unsigned size); - -struct io_hash_bucket { - spinlock_t lock; - struct hlist_head list; -} ____cacheline_aligned_in_smp; +void init_hash_table(struct io_hash_table *table, unsigned size); diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index f941c73f5502..344e7d90d557 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -158,8 +158,8 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, mutex_unlock(&ctx->uring_lock); seq_puts(m, "PollList:\n"); - for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { - struct io_hash_bucket *hb = &ctx->cancel_hash[i]; + for (i = 0; i < (1U << ctx->cancel_table.hash_bits); i++) { + struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; struct io_kiocb *req; spin_lock(&hb->lock); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1308a1f0c349..46db51641069 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -241,11 +241,23 @@ static __cold void io_fallback_req_func(struct work_struct *work) percpu_ref_put(&ctx->refs); } +static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits) +{ + unsigned hash_buckets = 1U << bits; + size_t hash_size = hash_buckets * sizeof(table->hbs[0]); + + table->hbs = kmalloc(hash_size, GFP_KERNEL); + if (!table->hbs) + return -ENOMEM; + + table->hash_bits = bits; + init_hash_table(table, hash_buckets); + return 0; +} + static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) { struct io_ring_ctx *ctx; - unsigned hash_buckets; - size_t hash_size; int hash_bits; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); @@ -261,16 +273,9 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) */ hash_bits = ilog2(p->cq_entries) - 5; hash_bits = clamp(hash_bits, 1, 8); - hash_buckets = 1U << hash_bits; - hash_size = hash_buckets * sizeof(struct io_hash_bucket); - - ctx->cancel_hash_bits = hash_bits; - ctx->cancel_hash = kmalloc(hash_size, GFP_KERNEL); - if (!ctx->cancel_hash) + if (io_alloc_hash_table(&ctx->cancel_table, hash_bits)) goto err; - init_hash_table(ctx->cancel_hash, hash_buckets); - ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL); if (!ctx->dummy_ubuf) goto err; @@ -311,7 +316,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) return ctx; err: kfree(ctx->dummy_ubuf); - kfree(ctx->cancel_hash); + kfree(ctx->cancel_table.hbs); kfree(ctx->io_bl); xa_destroy(&ctx->io_bl_xa); kfree(ctx); @@ -2499,7 +2504,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_req_caches_free(ctx); if (ctx->hash_map) io_wq_put_hash(ctx->hash_map); - kfree(ctx->cancel_hash); + kfree(ctx->cancel_table.hbs); kfree(ctx->dummy_ubuf); kfree(ctx->io_bl); xa_destroy(&ctx->io_bl_xa); diff --git a/io_uring/io_uring_types.h b/io_uring/io_uring_types.h index 8b00243abf65..d3b9bde9c702 100644 --- a/io_uring/io_uring_types.h +++ b/io_uring/io_uring_types.h @@ -9,6 +9,16 @@ #include "io-wq.h" #include "filetable.h" +struct io_hash_bucket { + spinlock_t lock; + struct hlist_head list; +} ____cacheline_aligned_in_smp; + +struct io_hash_table { + struct io_hash_bucket *hbs; + unsigned hash_bits; +}; + struct io_uring { u32 head ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp; @@ -224,8 +234,7 @@ struct io_ring_ctx { * manipulate the list, hence no extra locking is needed there. */ struct io_wq_work_list iopoll_list; - struct io_hash_bucket *cancel_hash; - unsigned cancel_hash_bits; + struct io_hash_table cancel_table; bool poll_multi_queue; struct list_head io_buffers_comp; diff --git a/io_uring/poll.c b/io_uring/poll.c index 5cc03be365e3..9c7793f5e93b 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -73,9 +73,9 @@ static struct io_poll *io_poll_get_single(struct io_kiocb *req) static void io_poll_req_insert(struct io_kiocb *req) { - struct io_ring_ctx *ctx = req->ctx; - u32 index = hash_long(req->cqe.user_data, ctx->cancel_hash_bits); - struct io_hash_bucket *hb = &ctx->cancel_hash[index]; + struct io_hash_table *table = &req->ctx->cancel_table; + u32 index = hash_long(req->cqe.user_data, table->hash_bits); + struct io_hash_bucket *hb = &table->hbs[index]; spin_lock(&hb->lock); hlist_add_head(&req->hash_node, &hb->list); @@ -84,8 +84,9 @@ static void io_poll_req_insert(struct io_kiocb *req) static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx) { - u32 index = hash_long(req->cqe.user_data, ctx->cancel_hash_bits); - spinlock_t *lock = &ctx->cancel_hash[index].lock; + struct io_hash_table *table = &req->ctx->cancel_table; + u32 index = hash_long(req->cqe.user_data, table->hash_bits); + spinlock_t *lock = &table->hbs[index].lock; spin_lock(lock); hash_del(&req->hash_node); @@ -533,13 +534,15 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, bool cancel_all) { + struct io_hash_table *table = &ctx->cancel_table; + unsigned nr_buckets = 1U << table->hash_bits; struct hlist_node *tmp; struct io_kiocb *req; bool found = false; int i; - for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { - struct io_hash_bucket *hb = &ctx->cancel_hash[i]; + for (i = 0; i < nr_buckets; i++) { + struct io_hash_bucket *hb = &table->hbs[i]; spin_lock(&hb->lock); hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { @@ -556,12 +559,12 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, struct io_cancel_data *cd, - struct io_hash_bucket hash_table[], + struct io_hash_table *table, struct io_hash_bucket **out_bucket) { struct io_kiocb *req; - u32 index = hash_long(cd->data, ctx->cancel_hash_bits); - struct io_hash_bucket *hb = &hash_table[index]; + u32 index = hash_long(cd->data, table->hash_bits); + struct io_hash_bucket *hb = &table->hbs[index]; *out_bucket = NULL; @@ -585,16 +588,17 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, struct io_cancel_data *cd, - struct io_hash_bucket hash_table[], + struct io_hash_table *table, struct io_hash_bucket **out_bucket) { + unsigned nr_buckets = 1U << table->hash_bits; struct io_kiocb *req; int i; *out_bucket = NULL; - for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { - struct io_hash_bucket *hb = &hash_table[i]; + for (i = 0; i < nr_buckets; i++) { + struct io_hash_bucket *hb = &table->hbs[i]; spin_lock(&hb->lock); hlist_for_each_entry(req, &hb->list, hash_node) { @@ -622,15 +626,15 @@ static bool io_poll_disarm(struct io_kiocb *req) } static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, - struct io_hash_bucket hash_table[]) + struct io_hash_table *table) { struct io_hash_bucket *bucket; struct io_kiocb *req; if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_ANY)) - req = io_poll_file_find(ctx, cd, ctx->cancel_hash, &bucket); + req = io_poll_file_find(ctx, cd, table, &bucket); else - req = io_poll_find(ctx, false, cd, ctx->cancel_hash, &bucket); + req = io_poll_find(ctx, false, cd, table, &bucket); if (req) io_poll_cancel_req(req); @@ -641,7 +645,7 @@ static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) { - return __io_poll_cancel(ctx, cd, ctx->cancel_hash); + return __io_poll_cancel(ctx, cd, &ctx->cancel_table); } static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, @@ -739,7 +743,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) int ret2, ret = 0; bool locked; - preq = io_poll_find(ctx, true, &cd, ctx->cancel_hash, &bucket); + preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table, &bucket); if (preq) ret2 = io_poll_disarm(preq); if (bucket) From patchwork Thu Jun 16 09:22:11 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883648 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 068C7CCA47A for ; Thu, 16 Jun 2022 09:23:05 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376658AbiFPJXE (ORCPT ); Thu, 16 Jun 2022 05:23:04 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47824 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376662AbiFPJXC (ORCPT ); Thu, 16 Jun 2022 05:23:02 -0400 Received: from mail-wm1-x331.google.com (mail-wm1-x331.google.com [IPv6:2a00:1450:4864:20::331]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E91531C90A for ; Thu, 16 Jun 2022 02:23:00 -0700 (PDT) Received: by mail-wm1-x331.google.com with SMTP id z17so430734wmi.1 for ; Thu, 16 Jun 2022 02:23:00 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=FnpmHNOzLmPzZqlagM4s9zuVri1qJ9NLqujQ99u3Owo=; b=XWnFDQU0Gi/azSH6aKGhHo/JPuqEDbU2VLToGd5eFqPHRErjfXajRLLSmigHGyI/CV WZ1aJqXnky/ENobi7VxM9zVb+MaUqbWONysHgotL3WX/kfe5vESQ2AJpUm9PWaEh1M/b KWvanOI8s7VU7zn/7KWjo4QNVFeLWMjuQ9yWL1PJYkt7HBZSZlhf0Bdl4IB4z4bY5/fi vsY91kGVg5hn0Wr8p+NFMM/xsV4WianZcjJHxOelD8OO5Z2OWSezXJaAcIbN7skFhiVm Fj5c3/Y8OJQO8oe3bOEtGJz2cpIXzXxLd/YfuGLjdFMNa9gbnl+njUu6M2LUodBBPC7U Eo0A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=FnpmHNOzLmPzZqlagM4s9zuVri1qJ9NLqujQ99u3Owo=; b=OaSKRo+fhMueKNEviNoKCOmMDPtLGN1Iqid7FB9MEZzO5zGsej90Bvu6RYWZKgSwRl zsaJo3u3FL2R89UWCnJshSgkYb5vbxdbP8c5l8byDreeZl/Ebf5gaEroYT65t+J61zKL esDsvcvrFzWmyXc8jhF5bMAeQUWrZXL8+aaekejJ1wjdIxIFz9fuY3B7gL4DrodiNCgk vNiH2q2xdqXQg4yJlhevNAje3XH9a1GgB81gX0p44TUs8q7ZTh2tibhXz1rlRIxJ/JzB Akml9/R+v0bc9DxcPK7FZtQB37veruQXto2oIE68FhTCiaEXiEj/VSGYR+itonni0HQL hjnw== X-Gm-Message-State: AJIora85lVuQdQJx1Z4wQEHPehbB/sYt17xfnLf1TKfsE4MPAl64XtgZ RJBuU6yABGb45yf1C6Ne37jSUsUhC4RVcQ== X-Google-Smtp-Source: AGRyM1v06kNkBWjrCIbQbTIv2A326E/5I3M2RlnVVJofY4hGOB5l53dCV0kUFjAChJX2X6zU10rfYg== X-Received: by 2002:a05:600c:3510:b0:39c:7fe7:cbd3 with SMTP id h16-20020a05600c351000b0039c7fe7cbd3mr4037494wmq.191.1655371379275; Thu, 16 Jun 2022 02:22:59 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.58 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:58 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 15/16] io_uring: propagate locking state to poll cancel Date: Thu, 16 Jun 2022 10:22:11 +0100 Message-Id: X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Poll cancellation will be soon need to grab ->uring_lock inside, pass the locking state, i.e. issue_flags, inside the cancellation functions. Signed-off-by: Pavel Begunkov --- io_uring/cancel.c | 7 ++++--- io_uring/cancel.h | 3 ++- io_uring/poll.c | 3 ++- io_uring/poll.h | 3 ++- io_uring/timeout.c | 3 ++- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/io_uring/cancel.c b/io_uring/cancel.c index f28f0a7d1272..f07bfd27c98a 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -78,7 +78,8 @@ static int io_async_cancel_one(struct io_uring_task *tctx, return ret; } -int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd) +int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd, + unsigned issue_flags) { struct io_ring_ctx *ctx = req->ctx; int ret; @@ -93,7 +94,7 @@ int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd) if (!ret) return 0; - ret = io_poll_cancel(ctx, cd); + ret = io_poll_cancel(ctx, cd, issue_flags); if (ret != -ENOENT) return ret; @@ -136,7 +137,7 @@ static int __io_async_cancel(struct io_cancel_data *cd, struct io_kiocb *req, int ret, nr = 0; do { - ret = io_try_cancel(req, cd); + ret = io_try_cancel(req, cd, issue_flags); if (ret == -ENOENT) break; if (!all) diff --git a/io_uring/cancel.h b/io_uring/cancel.h index fd4cb1a2595d..8dd259dc383e 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -3,5 +3,6 @@ int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags); -int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd); +int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd, + unsigned int issue_flags); void init_hash_table(struct io_hash_table *table, unsigned size); diff --git a/io_uring/poll.c b/io_uring/poll.c index 9c7793f5e93b..07157da1c2cb 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -643,7 +643,8 @@ static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, return req ? 0 : -ENOENT; } -int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) +int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, + unsigned issue_flags) { return __io_poll_cancel(ctx, cd, &ctx->cancel_table); } diff --git a/io_uring/poll.h b/io_uring/poll.h index cc75c1567a84..fa3e19790281 100644 --- a/io_uring/poll.h +++ b/io_uring/poll.h @@ -24,7 +24,8 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags); int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags); -int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); +int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, + unsigned issue_flags); int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags); bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, bool cancel_all); diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 69cca42d6835..526fc8b2e3b6 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -262,6 +262,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) { + unsigned issue_flags = *locked ? 0 : IO_URING_F_UNLOCKED; struct io_timeout *timeout = io_kiocb_to_cmd(req); struct io_kiocb *prev = timeout->prev; int ret = -ENOENT; @@ -273,7 +274,7 @@ static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) .data = prev->cqe.user_data, }; - ret = io_try_cancel(req, &cd); + ret = io_try_cancel(req, &cd, issue_flags); } io_req_set_res(req, ret ?: -ETIME, 0); io_req_complete_post(req); From patchwork Thu Jun 16 09:22:12 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12883649 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A6F22CCA47F for ; Thu, 16 Jun 2022 09:23:05 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376671AbiFPJXE (ORCPT ); Thu, 16 Jun 2022 05:23:04 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47872 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1376678AbiFPJXD (ORCPT ); Thu, 16 Jun 2022 05:23:03 -0400 Received: from mail-wm1-x335.google.com (mail-wm1-x335.google.com [IPv6:2a00:1450:4864:20::335]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 20E8E193E3 for ; Thu, 16 Jun 2022 02:23:02 -0700 (PDT) Received: by mail-wm1-x335.google.com with SMTP id c130-20020a1c3588000000b0039c6fd897b4so2485866wma.4 for ; Thu, 16 Jun 2022 02:23:02 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=fa43mkxPi6/7mBUtUl2b2eI4gg2Vxk77B9h3JQhQxf4=; b=TsEPJUwCWVJqy/flz217vTXDw+UHLHaCpeI5fi7f4R7vsMKKL1TRGT8qEGRftuXcWe E2EZH1z+O+AnsSQj38fakktZwCLvXJiw97d6hyVthDTcq1QKw21+kh2vBifiRuu82BmY 93iixr84ZPhBQCE1dNu2J45urVkCyzAFcozMckkfAQOETyB/KqZliBNrWup6vC32dLlp HEssC9Zc7+8pdbtMeZnWtJ9E77/nXJjYLy0tseS3e7pnF45bibU71WPJgLFCeEJMJZsg PBPiffacJmfik7dMB645LKAJb6igX0SaCtnNsG37WvGvO44w1bYKrJnXH5cV9zZzph+N Sqnw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=fa43mkxPi6/7mBUtUl2b2eI4gg2Vxk77B9h3JQhQxf4=; b=jj5XgwUCHMtlcOU8Jk+NELzmZIf6mqtwISGwMd+qJ9nw/rWBxo3myMEstoMea8f/U4 78xcy7eLcZ4jiQ8MrRf/ZXLm6o+41ToAkp5IYHHU6y4QHz3OFtYr+yusDaVk71nmjOh7 WklRGGDJZJH/+yW5yOMmHVdtj+izhLIw+uLTYvLltHmxBN9SAupOK5vcsH0K/RPiYkHR t8gPqcQeA75iJ6LBIQ38k0x1A/+cd/vvK2flP/Mo1Un8kwADL5yP+7DY+paqKoVsUs50 0vD0ERb0skkvHQeDij4gP0ORTUqhN2guWDZjRV4nDJdWjEv7X02VbZ8D7bhMz/UP7DLE 48Ig== X-Gm-Message-State: AJIora+mBy6vtU4bFmyTwXtgdMWxQnK4DdTK8Jd7PWZDOphWyg5vzUIT /txe0RerWdR1Sguj1jFsgeExdtK/ynEamQ== X-Google-Smtp-Source: AGRyM1t0x7Y+PPxrl2CGDN2EbTmKt81XgKiizhq9dRtU4bQFgK3rkVYpLOsf7CrtHWSAg2u4RjCRvQ== X-Received: by 2002:a05:600c:3cd:b0:39c:880e:dc7b with SMTP id z13-20020a05600c03cd00b0039c880edc7bmr3845664wmd.168.1655371380304; Thu, 16 Jun 2022 02:23:00 -0700 (PDT) Received: from 127.0.0.1localhost (188.28.125.106.threembb.co.uk. [188.28.125.106]) by smtp.gmail.com with ESMTPSA id s6-20020a1cf206000000b0039c975aa553sm1695221wmc.25.2022.06.16.02.22.59 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 16 Jun 2022 02:22:59 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH for-next v3 16/16] io_uring: mutex locked poll hashing Date: Thu, 16 Jun 2022 10:22:12 +0100 Message-Id: <1bbad9c78c454b7b92f100bbf46730a37df7194f.1655371007.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Currently we do two extra spin lock/unlock pairs to add a poll/apoll request to the cancellation hash table and remove it from there. On the submission side we often already hold ->uring_lock and tw completion is likely to hold it as well. Add a second cancellation hash table protected by ->uring_lock. In concerns for latency because of a need to have the mutex locked on the completion side, use the new table only in following cases: 1) IORING_SETUP_SINGLE_ISSUER: only one task grabs uring_lock, so there is little to no contention and so the main tw hander will almost always end up grabbing it before calling callbacks. 2) IORING_SETUP_SQPOLL: same as with single issuer, only one task is a major user of ->uring_lock. 3) apoll: we normally grab the lock on the completion side anyway to execute the request, so it's free. Signed-off-by: Pavel Begunkov --- io_uring/io_uring.c | 9 ++- io_uring/io_uring_types.h | 4 ++ io_uring/poll.c | 117 +++++++++++++++++++++++++++++++------- 3 files changed, 108 insertions(+), 22 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 46db51641069..d256a611be4e 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -275,6 +275,8 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) hash_bits = clamp(hash_bits, 1, 8); if (io_alloc_hash_table(&ctx->cancel_table, hash_bits)) goto err; + if (io_alloc_hash_table(&ctx->cancel_table_locked, hash_bits)) + goto err; ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL); if (!ctx->dummy_ubuf) @@ -317,6 +319,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) err: kfree(ctx->dummy_ubuf); kfree(ctx->cancel_table.hbs); + kfree(ctx->cancel_table_locked.hbs); kfree(ctx->io_bl); xa_destroy(&ctx->io_bl_xa); kfree(ctx); @@ -2505,6 +2508,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) if (ctx->hash_map) io_wq_put_hash(ctx->hash_map); kfree(ctx->cancel_table.hbs); + kfree(ctx->cancel_table_locked.hbs); kfree(ctx->dummy_ubuf); kfree(ctx->io_bl); xa_destroy(&ctx->io_bl_xa); @@ -2666,12 +2670,13 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) __io_cqring_overflow_flush(ctx, true); xa_for_each(&ctx->personalities, index, creds) io_unregister_personality(ctx, index); + if (ctx->rings) + io_poll_remove_all(ctx, NULL, true); mutex_unlock(&ctx->uring_lock); /* failed during ring init, it couldn't have issued any requests */ if (ctx->rings) { io_kill_timeouts(ctx, NULL, true); - io_poll_remove_all(ctx, NULL, true); /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); } @@ -2796,7 +2801,9 @@ static __cold void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } ret |= io_cancel_defer_files(ctx, task, cancel_all); + mutex_lock(&ctx->uring_lock); ret |= io_poll_remove_all(ctx, task, cancel_all); + mutex_unlock(&ctx->uring_lock); ret |= io_kill_timeouts(ctx, task, cancel_all); if (task) ret |= io_run_task_work(); diff --git a/io_uring/io_uring_types.h b/io_uring/io_uring_types.h index d3b9bde9c702..65ac7cdaaa73 100644 --- a/io_uring/io_uring_types.h +++ b/io_uring/io_uring_types.h @@ -191,6 +191,7 @@ struct io_ring_ctx { struct xarray io_bl_xa; struct list_head io_buffers_cache; + struct io_hash_table cancel_table_locked; struct list_head cq_overflow_list; struct list_head apoll_cache; struct xarray personalities; @@ -323,6 +324,7 @@ enum { REQ_F_CQE32_INIT_BIT, REQ_F_APOLL_MULTISHOT_BIT, REQ_F_CLEAR_POLLIN_BIT, + REQ_F_HASH_LOCKED_BIT, /* keep async read/write and isreg together and in order */ REQ_F_SUPPORT_NOWAIT_BIT, REQ_F_ISREG_BIT, @@ -393,6 +395,8 @@ enum { REQ_F_CQE32_INIT = BIT(REQ_F_CQE32_INIT_BIT), /* recvmsg special flag, clear EPOLLIN */ REQ_F_CLEAR_POLLIN = BIT(REQ_F_CLEAR_POLLIN_BIT), + /* hashed into ->cancel_hash_locked, protected by ->uring_lock */ + REQ_F_HASH_LOCKED = BIT(REQ_F_HASH_LOCKED_BIT), }; typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); diff --git a/io_uring/poll.c b/io_uring/poll.c index 07157da1c2cb..2e068e05732a 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -93,6 +93,32 @@ static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx) spin_unlock(lock); } +static void io_poll_req_insert_locked(struct io_kiocb *req) +{ + struct io_hash_table *table = &req->ctx->cancel_table_locked; + u32 index = hash_long(req->cqe.user_data, table->hash_bits); + + hlist_add_head(&req->hash_node, &table->hbs[index].list); +} + +static void io_poll_tw_hash_eject(struct io_kiocb *req, bool *locked) +{ + struct io_ring_ctx *ctx = req->ctx; + + if (req->flags & REQ_F_HASH_LOCKED) { + /* + * ->cancel_table_locked is protected by ->uring_lock in + * contrast to per bucket spinlocks. Likely, tctx_task_work() + * already grabbed the mutex for us, but there is a chance it + * failed. + */ + io_tw_lock(ctx, locked); + hash_del(&req->hash_node); + } else { + io_poll_req_delete(req, ctx); + } +} + static void io_init_poll_iocb(struct io_poll *poll, __poll_t events, wait_queue_func_t wake_func) { @@ -217,7 +243,6 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked) static void io_poll_task_func(struct io_kiocb *req, bool *locked) { - struct io_ring_ctx *ctx = req->ctx; int ret; ret = io_poll_check_events(req, locked); @@ -234,7 +259,8 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) } io_poll_remove_entries(req); - io_poll_req_delete(req, ctx); + io_poll_tw_hash_eject(req, locked); + io_req_set_res(req, req->cqe.res, 0); io_req_task_complete(req, locked); } @@ -248,7 +274,7 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) return; io_poll_remove_entries(req); - io_poll_req_delete(req, req->ctx); + io_poll_tw_hash_eject(req, locked); if (!ret) io_req_task_submit(req, locked); @@ -442,7 +468,10 @@ static int __io_arm_poll_handler(struct io_kiocb *req, return 0; } - io_poll_req_insert(req); + if (req->flags & REQ_F_HASH_LOCKED) + io_poll_req_insert_locked(req); + else + io_poll_req_insert(req); if (mask && (poll->events & EPOLLET)) { /* can't multishot if failed, just queue the event we've got */ @@ -480,6 +509,15 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) __poll_t mask = POLLPRI | POLLERR | EPOLLET; int ret; + /* + * apoll requests already grab the mutex to complete in the tw handler, + * so removal from the mutex-backed hash is free, use it by default. + */ + if (issue_flags & IO_URING_F_UNLOCKED) + req->flags &= ~REQ_F_HASH_LOCKED; + else + req->flags |= REQ_F_HASH_LOCKED; + if (!def->pollin && !def->pollout) return IO_APOLL_ABORTED; if (!file_can_poll(req->file)) @@ -528,13 +566,10 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) return IO_APOLL_OK; } -/* - * Returns true if we found and killed one or more poll requests - */ -__cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, - bool cancel_all) +static __cold bool io_poll_remove_all_table(struct task_struct *tsk, + struct io_hash_table *table, + bool cancel_all) { - struct io_hash_table *table = &ctx->cancel_table; unsigned nr_buckets = 1U << table->hash_bits; struct hlist_node *tmp; struct io_kiocb *req; @@ -557,6 +592,17 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, return found; } +/* + * Returns true if we found and killed one or more poll requests + */ +__cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, + bool cancel_all) + __must_hold(&ctx->uring_lock) +{ + return io_poll_remove_all_table(tsk, &ctx->cancel_table, cancel_all) | + io_poll_remove_all_table(tsk, &ctx->cancel_table_locked, cancel_all); +} + static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, struct io_cancel_data *cd, struct io_hash_table *table, @@ -616,13 +662,15 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, return NULL; } -static bool io_poll_disarm(struct io_kiocb *req) +static int io_poll_disarm(struct io_kiocb *req) { + if (!req) + return -ENOENT; if (!io_poll_get_ownership(req)) - return false; + return -EALREADY; io_poll_remove_entries(req); hash_del(&req->hash_node); - return true; + return 0; } static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, @@ -646,7 +694,16 @@ static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned issue_flags) { - return __io_poll_cancel(ctx, cd, &ctx->cancel_table); + int ret; + + ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table); + if (ret != -ENOENT) + return ret; + + io_ring_submit_lock(ctx, issue_flags); + ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table_locked); + io_ring_submit_unlock(ctx, issue_flags); + return ret; } static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, @@ -721,6 +778,16 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) ipt.pt._qproc = io_poll_queue_proc; + /* + * If sqpoll or single issuer, there is no contention for ->uring_lock + * and we'll end up holding it in tw handlers anyway. + */ + if (!(issue_flags & IO_URING_F_UNLOCKED) && + (req->ctx->flags & (IORING_SETUP_SQPOLL | IORING_SETUP_SINGLE_ISSUER))) + req->flags |= REQ_F_HASH_LOCKED; + else + req->flags &= ~REQ_F_HASH_LOCKED; + ret = __io_arm_poll_handler(req, poll, &ipt, poll->events); if (ipt.error) { return ipt.error; @@ -745,20 +812,28 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) bool locked; preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table, &bucket); - if (preq) - ret2 = io_poll_disarm(preq); + ret2 = io_poll_disarm(preq); if (bucket) spin_unlock(&bucket->lock); - - if (!preq) { - ret = -ENOENT; + if (!ret2) + goto found; + if (ret2 != -ENOENT) { + ret = ret2; goto out; } - if (!ret2) { - ret = -EALREADY; + + io_ring_submit_lock(ctx, issue_flags); + preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table_locked, &bucket); + ret2 = io_poll_disarm(preq); + if (bucket) + spin_unlock(&bucket->lock); + io_ring_submit_unlock(ctx, issue_flags); + if (ret2) { + ret = ret2; goto out; } +found: if (poll_update->update_events || poll_update->update_user_data) { /* only mask one event flags, keep behavior flags */ if (poll_update->update_events) {