| Message ID | 20231122134833.20825-6-aaptel@nvidia.com (mailing list archive) |
|---|---|
| State | Deferred |
| Headers | show |
| Series | nvme-tcp receive offloads | expand |
On 11/22/23 15:48, Aurelien Aptel wrote: > From: Boris Pismenny <borisp@nvidia.com> > > This commit introduces direct data placement offload to NVME > TCP. There is a context per queue, which is established after the > handshake using the sk_add/del NDOs. > > Additionally, a resynchronization routine is used to assist > hardware recovery from TCP OOO, and continue the offload. > Resynchronization operates as follows: > > 1. TCP OOO causes the NIC HW to stop the offload > > 2. NIC HW identifies a PDU header at some TCP sequence number, > and asks NVMe-TCP to confirm it. > This request is delivered from the NIC driver to NVMe-TCP by first > finding the socket for the packet that triggered the request, and > then finding the nvme_tcp_queue that is used by this routine. > Finally, the request is recorded in the nvme_tcp_queue. > > 3. When NVMe-TCP observes the requested TCP sequence, it will compare > it with the PDU header TCP sequence, and report the result to the > NIC driver (resync), which will update the HW, and resume offload > when all is successful. > > Some HW implementation such as ConnectX-7 assume linear CCID (0...N-1 > for queue of size N) where the linux nvme driver uses part of the 16 > bit CCID for generation counter. To address that, we use the existing > quirk in the nvme layer when the HW driver advertises if the device is > not supports the full 16 bit CCID range. > > Furthermore, we let the offloading driver advertise what is the max hw > sectors/segments via ulp_ddp_limits. > > A follow-up patch introduces the data-path changes required for this > offload. > > Socket operations need a netdev reference. This reference is > dropped on NETDEV_GOING_DOWN events to allow the device to go down in > a follow-up patch. > > Signed-off-by: Boris Pismenny <borisp@nvidia.com> > Signed-off-by: Ben Ben-Ishay <benishay@nvidia.com> > Signed-off-by: Or Gerlitz <ogerlitz@nvidia.com> > Signed-off-by: Yoray Zack <yorayz@nvidia.com> > Signed-off-by: Shai Malin <smalin@nvidia.com> > Signed-off-by: Aurelien Aptel <aaptel@nvidia.com> > Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com> > --- > drivers/nvme/host/tcp.c | 259 ++++++++++++++++++++++++++++++++++++++-- > 1 file changed, 246 insertions(+), 13 deletions(-) > > diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c > index 89661a9cf850..7ad6a4854fce 100644 > --- a/drivers/nvme/host/tcp.c > +++ b/drivers/nvme/host/tcp.c > @@ -21,6 +21,10 @@ > #include <net/busy_poll.h> > #include <trace/events/sock.h> > > +#ifdef CONFIG_ULP_DDP > +#include <net/ulp_ddp.h> > +#endif > + > #include "nvme.h" > #include "fabrics.h" > > @@ -46,6 +50,16 @@ MODULE_PARM_DESC(tls_handshake_timeout, > "nvme TLS handshake timeout in seconds (default 10)"); > #endif > > +#ifdef CONFIG_ULP_DDP > +/* NVMeTCP direct data placement and data digest offload will not > + * happen if this parameter false (default), regardless of what the > + * underlying netdev capabilities are. > + */ > +static bool ddp_offload; > +module_param(ddp_offload, bool, 0644); > +MODULE_PARM_DESC(ddp_offload, "Enable or disable NVMeTCP direct data placement support"); > +#endif > + > #ifdef CONFIG_DEBUG_LOCK_ALLOC > /* lockdep can detect a circular dependency of the form > * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock > @@ -119,6 +133,7 @@ enum nvme_tcp_queue_flags { > NVME_TCP_Q_ALLOCATED = 0, > NVME_TCP_Q_LIVE = 1, > NVME_TCP_Q_POLLING = 2, > + NVME_TCP_Q_OFF_DDP = 3, > }; > > enum nvme_tcp_recv_state { > @@ -146,6 +161,18 @@ struct nvme_tcp_queue { > size_t ddgst_remaining; > unsigned int nr_cqe; > > +#ifdef CONFIG_ULP_DDP > + /* > + * resync_tcp_seq is a speculative PDU header tcp seq number (with > + * an additional flag in the lower 32 bits) that the HW send to > + * the SW, for the SW to verify. > + * - The 32 high bits store the seq number > + * - The 32 low bits are used as a flag to know if a request > + * is pending (ULP_DDP_RESYNC_PENDING). > + */ > + atomic64_t resync_tcp_seq; > +#endif > + > /* send state */ > struct nvme_tcp_request *request; > > @@ -188,6 +215,12 @@ struct nvme_tcp_ctrl { > struct delayed_work connect_work; > struct nvme_tcp_request async_req; > u32 io_queues[HCTX_MAX_TYPES]; > + > + struct net_device *ddp_netdev; > + u32 ddp_threshold; > +#ifdef CONFIG_ULP_DDP > + struct ulp_ddp_limits ddp_limits; > +#endif > }; > > static LIST_HEAD(nvme_tcp_ctrl_list); > @@ -291,6 +324,166 @@ static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, > return nvme_tcp_pdu_data_left(req) <= len; > } > > +#ifdef CONFIG_ULP_DDP > + > +static struct net_device * > +nvme_tcp_get_ddp_netdev_with_limits(struct nvme_tcp_ctrl *ctrl) > +{ > + struct net_device *netdev; > + bool ok; > + > + if (!ddp_offload) > + return NULL; > + > + /* netdev ref is put in nvme_tcp_stop_admin_queue() */ > + netdev = get_netdev_for_sock(ctrl->queues[0].sock->sk); > + if (!netdev) { > + dev_dbg(ctrl->ctrl.device, "netdev not found\n"); > + return NULL; > + } > + > + ok = ulp_ddp_query_limits(netdev, &ctrl->ddp_limits, > + ULP_DDP_NVME, ULP_DDP_CAP_NVME_TCP, > + ctrl->ctrl.opts->tls); > + if (!ok) { please use a normal name (ret). Plus, its strange that a query function receives a feature and returns true/false based on this. The query should return the limits, and the caller should look at the limits and see if it is appropriately supported. The rest looks fine I think.
Hi Sagi, Sagi Grimberg <sagi@grimberg.me> writes: >> + ok = ulp_ddp_query_limits(netdev, &ctrl->ddp_limits, >> + ULP_DDP_NVME, ULP_DDP_CAP_NVME_TCP, >> + ctrl->ctrl.opts->tls); >> + if (!ok) { > > please use a normal name (ret). Ok, we will rename to ret and make ulp_ddp_query_limits() return int 0 on success to be consistent with the name. > Plus, its strange that a query function receives a feature and returns > true/false based on this. The query should return the limits, and the > caller should look at the limits and see if it is appropriately > supported. We are not sure how to proceed as this seems to conflict with what you suggested in v12 [1] about hiding the details of checking supports in the API. Limits just dictate some constants the nvme-layer should use once we know it is supported. We can rename ulp_ddp_query_limits() to ulp_ddp_check_support(). This function checks the support of the specified offload capability and also returns the limitations of it. Alternatively, we can split it in 2 API functions (check_support and query_limits). Let us know what you prefer. Thanks 1: https://lkml.kernel.org/netdev/bc5cd2a7-efc4-e4df-cae5-5c527dd704a6@grimberg.me/
Hi Sagi, On 29/11/2023 15:52, Aurelien Aptel wrote: > Hi Sagi, > > Sagi Grimberg <sagi@grimberg.me> writes: >>> + ok = ulp_ddp_query_limits(netdev, &ctrl->ddp_limits, >>> + ULP_DDP_NVME, ULP_DDP_CAP_NVME_TCP, >>> + ctrl->ctrl.opts->tls); >>> + if (!ok) { >> >> please use a normal name (ret). > > Ok, we will rename to ret and make ulp_ddp_query_limits() return int 0 > on success to be consistent with the name. > >> Plus, its strange that a query function receives a feature and returns >> true/false based on this. The query should return the limits, and the >> caller should look at the limits and see if it is appropriately >> supported. > > We are not sure how to proceed as this seems to conflict with what you > suggested in v12 [1] about hiding the details of checking supports in > the API. Limits just dictate some constants the nvme-layer should use > once we know it is supported. > > We can rename ulp_ddp_query_limits() to ulp_ddp_check_support(). This > function checks the support of the specified offload capability and also > returns the limitations of it. > > Alternatively, we can split it in 2 API functions (check_support > and query_limits). > > Let us know what you prefer. > > Thanks > > 1: https://lkml.kernel.org/netdev/bc5cd2a7-efc4-e4df-cae5-5c527dd704a6@grimberg.me/ We would like to submit another version by the end of this week and hopefully progress with this series. To address your comment, I was thinking about something like: + if (!ulp_ddp_is_cap_active(netdev, ULP_DDP_CAP_NVME_TCP)) + goto err; + + ret = ulp_ddp_get_limits(netdev, &ctrl->ddp_limits, ULP_DDP_NVME); + if (ret) + goto err; + + if (ctrl->ctrl.opts->tls && !ctrl->ddp_limits.tls) + goto err; I would like to remind you that the community didn't want to add ulp_ddp caps and limits context to the netdev structure (as we have for example in the block layer q) and preferred the design of ops/cbs.
> Hi Sagi, > > On 29/11/2023 15:52, Aurelien Aptel wrote: >> Hi Sagi, >> >> Sagi Grimberg <sagi@grimberg.me> writes: >>>> + ok = ulp_ddp_query_limits(netdev, &ctrl->ddp_limits, >>>> + ULP_DDP_NVME, ULP_DDP_CAP_NVME_TCP, >>>> + ctrl->ctrl.opts->tls); >>>> + if (!ok) { >>> >>> please use a normal name (ret). >> >> Ok, we will rename to ret and make ulp_ddp_query_limits() return int 0 >> on success to be consistent with the name. >> >>> Plus, its strange that a query function receives a feature and returns >>> true/false based on this. The query should return the limits, and the >>> caller should look at the limits and see if it is appropriately >>> supported. >> >> We are not sure how to proceed as this seems to conflict with what you >> suggested in v12 [1] about hiding the details of checking supports in >> the API. Limits just dictate some constants the nvme-layer should use >> once we know it is supported. >> >> We can rename ulp_ddp_query_limits() to ulp_ddp_check_support(). This >> function checks the support of the specified offload capability and also >> returns the limitations of it. >> >> Alternatively, we can split it in 2 API functions (check_support >> and query_limits). >> >> Let us know what you prefer. >> >> Thanks >> >> 1: >> https://lkml.kernel.org/netdev/bc5cd2a7-efc4-e4df-cae5-5c527dd704a6@grimberg.me/ > > We would like to submit another version by the end of this week and > hopefully progress with this series. > To address your comment, I was thinking about something like: > > + if (!ulp_ddp_is_cap_active(netdev, ULP_DDP_CAP_NVME_TCP)) > + goto err; > + > + ret = ulp_ddp_get_limits(netdev, &ctrl->ddp_limits, ULP_DDP_NVME); > + if (ret) > + goto err; > + > + if (ctrl->ctrl.opts->tls && !ctrl->ddp_limits.tls) > + goto err; > > I would like to remind you that the community didn't want to add ulp_ddp > caps and limits context to the netdev structure (as we have for example > in the block layer q) and preferred the design of ops/cbs. Seems fine to me.
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 89661a9cf850..7ad6a4854fce 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -21,6 +21,10 @@ #include <net/busy_poll.h> #include <trace/events/sock.h> +#ifdef CONFIG_ULP_DDP +#include <net/ulp_ddp.h> +#endif + #include "nvme.h" #include "fabrics.h" @@ -46,6 +50,16 @@ MODULE_PARM_DESC(tls_handshake_timeout, "nvme TLS handshake timeout in seconds (default 10)"); #endif +#ifdef CONFIG_ULP_DDP +/* NVMeTCP direct data placement and data digest offload will not + * happen if this parameter false (default), regardless of what the + * underlying netdev capabilities are. + */ +static bool ddp_offload; +module_param(ddp_offload, bool, 0644); +MODULE_PARM_DESC(ddp_offload, "Enable or disable NVMeTCP direct data placement support"); +#endif + #ifdef CONFIG_DEBUG_LOCK_ALLOC /* lockdep can detect a circular dependency of the form * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock @@ -119,6 +133,7 @@ enum nvme_tcp_queue_flags { NVME_TCP_Q_ALLOCATED = 0, NVME_TCP_Q_LIVE = 1, NVME_TCP_Q_POLLING = 2, + NVME_TCP_Q_OFF_DDP = 3, }; enum nvme_tcp_recv_state { @@ -146,6 +161,18 @@ struct nvme_tcp_queue { size_t ddgst_remaining; unsigned int nr_cqe; +#ifdef CONFIG_ULP_DDP + /* + * resync_tcp_seq is a speculative PDU header tcp seq number (with + * an additional flag in the lower 32 bits) that the HW send to + * the SW, for the SW to verify. + * - The 32 high bits store the seq number + * - The 32 low bits are used as a flag to know if a request + * is pending (ULP_DDP_RESYNC_PENDING). + */ + atomic64_t resync_tcp_seq; +#endif + /* send state */ struct nvme_tcp_request *request; @@ -188,6 +215,12 @@ struct nvme_tcp_ctrl { struct delayed_work connect_work; struct nvme_tcp_request async_req; u32 io_queues[HCTX_MAX_TYPES]; + + struct net_device *ddp_netdev; + u32 ddp_threshold; +#ifdef CONFIG_ULP_DDP + struct ulp_ddp_limits ddp_limits; +#endif }; static LIST_HEAD(nvme_tcp_ctrl_list); @@ -291,6 +324,166 @@ static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, return nvme_tcp_pdu_data_left(req) <= len; } +#ifdef CONFIG_ULP_DDP + +static struct net_device * +nvme_tcp_get_ddp_netdev_with_limits(struct nvme_tcp_ctrl *ctrl) +{ + struct net_device *netdev; + bool ok; + + if (!ddp_offload) + return NULL; + + /* netdev ref is put in nvme_tcp_stop_admin_queue() */ + netdev = get_netdev_for_sock(ctrl->queues[0].sock->sk); + if (!netdev) { + dev_dbg(ctrl->ctrl.device, "netdev not found\n"); + return NULL; + } + + ok = ulp_ddp_query_limits(netdev, &ctrl->ddp_limits, + ULP_DDP_NVME, ULP_DDP_CAP_NVME_TCP, + ctrl->ctrl.opts->tls); + if (!ok) { + dev_put(netdev); + return NULL; + } + + return netdev; +} + +static bool nvme_tcp_resync_request(struct sock *sk, u32 seq, u32 flags); +static const struct ulp_ddp_ulp_ops nvme_tcp_ddp_ulp_ops = { + .resync_request = nvme_tcp_resync_request, +}; + +static int nvme_tcp_offload_socket(struct nvme_tcp_queue *queue) +{ + struct ulp_ddp_config config = {.type = ULP_DDP_NVME}; + int ret; + + config.nvmeotcp.pfv = NVME_TCP_PFV_1_0; + config.nvmeotcp.cpda = 0; + config.nvmeotcp.dgst = + queue->hdr_digest ? NVME_TCP_HDR_DIGEST_ENABLE : 0; + config.nvmeotcp.dgst |= + queue->data_digest ? NVME_TCP_DATA_DIGEST_ENABLE : 0; + config.nvmeotcp.queue_size = queue->ctrl->ctrl.sqsize + 1; + config.nvmeotcp.queue_id = nvme_tcp_queue_id(queue); + + ret = ulp_ddp_sk_add(queue->ctrl->ddp_netdev, + queue->sock->sk, + &config, + &nvme_tcp_ddp_ulp_ops); + if (ret) + return ret; + + set_bit(NVME_TCP_Q_OFF_DDP, &queue->flags); + + return 0; +} + +static void nvme_tcp_unoffload_socket(struct nvme_tcp_queue *queue) +{ + clear_bit(NVME_TCP_Q_OFF_DDP, &queue->flags); + ulp_ddp_sk_del(queue->ctrl->ddp_netdev, queue->sock->sk); +} + +static void nvme_tcp_ddp_apply_limits(struct nvme_tcp_ctrl *ctrl) +{ + ctrl->ctrl.max_segments = ctrl->ddp_limits.max_ddp_sgl_len; + ctrl->ctrl.max_hw_sectors = + ctrl->ddp_limits.max_ddp_sgl_len << (ilog2(SZ_4K) - SECTOR_SHIFT); + ctrl->ddp_threshold = ctrl->ddp_limits.io_threshold; + + /* offloading HW doesn't support full ccid range, apply the quirk */ + ctrl->ctrl.quirks |= + ctrl->ddp_limits.nvmeotcp.full_ccid_range ? 0 : NVME_QUIRK_SKIP_CID_GEN; +} + +/* In presence of packet drops or network packet reordering, the device may lose + * synchronization between the TCP stream and the L5P framing, and require a + * resync with the kernel's TCP stack. + * + * - NIC HW identifies a PDU header at some TCP sequence number, + * and asks NVMe-TCP to confirm it. + * - When NVMe-TCP observes the requested TCP sequence, it will compare + * it with the PDU header TCP sequence, and report the result to the + * NIC driver + */ +static void nvme_tcp_resync_response(struct nvme_tcp_queue *queue, + struct sk_buff *skb, unsigned int offset) +{ + u64 pdu_seq = TCP_SKB_CB(skb)->seq + offset - queue->pdu_offset; + struct net_device *netdev = queue->ctrl->ddp_netdev; + u64 pdu_val = (pdu_seq << 32) | ULP_DDP_RESYNC_PENDING; + u64 resync_val; + u32 resync_seq; + + resync_val = atomic64_read(&queue->resync_tcp_seq); + /* Lower 32 bit flags. Check validity of the request */ + if ((resync_val & ULP_DDP_RESYNC_PENDING) == 0) + return; + + /* + * Obtain and check requested sequence number: is this PDU header + * before the request? + */ + resync_seq = resync_val >> 32; + if (before(pdu_seq, resync_seq)) + return; + + /* + * The atomic operation guarantees that we don't miss any NIC driver + * resync requests submitted after the above checks. + */ + if (atomic64_cmpxchg(&queue->resync_tcp_seq, pdu_val, + pdu_val & ~ULP_DDP_RESYNC_PENDING) != + atomic64_read(&queue->resync_tcp_seq)) + ulp_ddp_resync(netdev, queue->sock->sk, pdu_seq); +} + +static bool nvme_tcp_resync_request(struct sock *sk, u32 seq, u32 flags) +{ + struct nvme_tcp_queue *queue = sk->sk_user_data; + + /* + * "seq" (TCP seq number) is what the HW assumes is the + * beginning of a PDU. The nvme-tcp layer needs to store the + * number along with the "flags" (ULP_DDP_RESYNC_PENDING) to + * indicate that a request is pending. + */ + atomic64_set(&queue->resync_tcp_seq, (((uint64_t)seq << 32) | flags)); + + return true; +} + +#else + +static struct net_device * +nvme_tcp_get_ddp_netdev_with_limits(struct nvme_tcp_ctrl *ctrl) +{ + return NULL; +} + +static void nvme_tcp_ddp_apply_limits(struct nvme_tcp_ctrl *ctrl) +{} + +static int nvme_tcp_offload_socket(struct nvme_tcp_queue *queue) +{ + return 0; +} + +static void nvme_tcp_unoffload_socket(struct nvme_tcp_queue *queue) +{} + +static void nvme_tcp_resync_response(struct nvme_tcp_queue *queue, + struct sk_buff *skb, unsigned int offset) +{} + +#endif + static void nvme_tcp_init_iter(struct nvme_tcp_request *req, unsigned int dir) { @@ -733,6 +926,9 @@ static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); int ret; + if (test_bit(NVME_TCP_Q_OFF_DDP, &queue->flags)) + nvme_tcp_resync_response(queue, skb, *offset); + ret = skb_copy_bits(skb, *offset, &pdu[queue->pdu_offset], rcv_len); if (unlikely(ret)) @@ -1807,6 +2003,8 @@ static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) kernel_sock_shutdown(queue->sock, SHUT_RDWR); nvme_tcp_restore_sock_ops(queue); cancel_work_sync(&queue->io_work); + if (test_bit(NVME_TCP_Q_OFF_DDP, &queue->flags)) + nvme_tcp_unoffload_socket(queue); } static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) @@ -1823,6 +2021,20 @@ static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) mutex_unlock(&queue->queue_lock); } +static void nvme_tcp_stop_admin_queue(struct nvme_ctrl *nctrl) +{ + struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); + + nvme_tcp_stop_queue(nctrl, 0); + + /* + * We are called twice by nvme_tcp_teardown_admin_queue() + * Set ddp_netdev to NULL to avoid putting it twice + */ + dev_put(ctrl->ddp_netdev); + ctrl->ddp_netdev = NULL; +} + static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue) { write_lock_bh(&queue->sock->sk->sk_callback_lock); @@ -1849,19 +2061,37 @@ static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) nvme_tcp_init_recv_ctx(queue); nvme_tcp_setup_sock_ops(queue); - if (idx) + if (idx) { ret = nvmf_connect_io_queue(nctrl, idx); - else + if (ret) + goto err; + + if (ctrl->ddp_netdev) { + ret = nvme_tcp_offload_socket(queue); + if (ret) { + dev_info(nctrl->device, + "failed to setup offload on queue %d ret=%d\n", + idx, ret); + } + } + } else { ret = nvmf_connect_admin_queue(nctrl); + if (ret) + goto err; + + ctrl->ddp_netdev = nvme_tcp_get_ddp_netdev_with_limits(ctrl); + if (ctrl->ddp_netdev) + nvme_tcp_ddp_apply_limits(ctrl); - if (!ret) { - set_bit(NVME_TCP_Q_LIVE, &queue->flags); - } else { - if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) - __nvme_tcp_stop_queue(queue); - dev_err(nctrl->device, - "failed to connect queue: %d ret=%d\n", idx, ret); } + + set_bit(NVME_TCP_Q_LIVE, &queue->flags); + return 0; +err: + if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) + __nvme_tcp_stop_queue(queue); + dev_err(nctrl->device, + "failed to connect queue: %d ret=%d\n", idx, ret); return ret; } @@ -2073,7 +2303,7 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) { - nvme_tcp_stop_queue(ctrl, 0); + nvme_tcp_stop_admin_queue(ctrl); if (remove) nvme_remove_admin_tag_set(ctrl); nvme_tcp_free_admin_queue(ctrl); @@ -2116,7 +2346,7 @@ static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) nvme_quiesce_admin_queue(ctrl); blk_sync_queue(ctrl->admin_q); out_stop_queue: - nvme_tcp_stop_queue(ctrl, 0); + nvme_tcp_stop_admin_queue(ctrl); nvme_cancel_admin_tagset(ctrl); out_cleanup_tagset: if (new) @@ -2131,7 +2361,7 @@ static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, { nvme_quiesce_admin_queue(ctrl); blk_sync_queue(ctrl->admin_q); - nvme_tcp_stop_queue(ctrl, 0); + nvme_tcp_stop_admin_queue(ctrl); nvme_cancel_admin_tagset(ctrl); if (remove) nvme_unquiesce_admin_queue(ctrl); @@ -2415,7 +2645,10 @@ static void nvme_tcp_complete_timed_out(struct request *rq) struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; - nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); + if (nvme_tcp_admin_queue(req->queue)) + nvme_tcp_stop_admin_queue(ctrl); + else + nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); nvmf_complete_timed_out_request(rq); }