| Message ID | 20190225051204.3784-2-ira.weiny@intel.com (mailing list archive) |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series | [V5,1/6] IB/MAD: Add send path trace points | expand |
On Sun, 24 Feb 2019 21:11:59 -0800 ira.weiny@intel.com wrote: > INGENIC JZ4780 DMA Driver > M: Zubair Lutfullah Kakakhel <Zubair.Kakakhel@imgtec.com> > diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c > index e742a6a2c138..d5cd90fecda7 100644 > --- a/drivers/infiniband/core/mad.c > +++ b/drivers/infiniband/core/mad.c > @@ -3,7 +3,7 @@ > * Copyright (c) 2005 Intel Corporation. All rights reserved. > * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. > * Copyright (c) 2009 HNR Consulting. All rights reserved. > - * Copyright (c) 2014 Intel Corporation. All rights reserved. > + * Copyright (c) 2014,2018 Intel Corporation. All rights reserved. > * > * This software is available to you under a choice of one of two > * licenses. You may choose to be licensed under the terms of the GNU > @@ -51,6 +51,41 @@ > #include "opa_smi.h" > #include "agent.h" > > +static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, > + struct ib_mad_qp_info *qp_info, > + u32 *dlid, u8 *sl, u16 *pkey, u32 *rqpn, > + u32 *rqkey) > +{ Instead of passing in a bunch of addresses, you may be able to pass in the entry structure directly. struct trace_event_raw_ib_mad_send_template *entry (defined from DECLARE_EVENT_CLASS(name, ... struct trace_event_raw_##name .. ) And just assign everything as: entry->dlid = x, ... -- Steve > + struct ib_device *dev = qp_info->port_priv->device; > + u8 pnum = qp_info->port_priv->port_num; > + struct ib_ud_wr *wr = &mad_send_wr->send_wr; > + struct rdma_ah_attr attr; > + > + memset(&attr, 0, sizeof(attr)); > + rdma_query_ah(wr->ah, &attr); > + > + /* These are common */ > + *sl = attr.sl; > + ib_query_pkey(dev, pnum, wr->pkey_index, pkey); > + *rqpn = wr->remote_qpn; > + *rqkey = wr->remote_qkey; > + > + switch (attr.type) { > + case RDMA_AH_ATTR_TYPE_IB: > + *dlid = attr.ib.dlid; > + break; > + case RDMA_AH_ATTR_TYPE_OPA: > + *dlid = attr.opa.dlid; > + break; > + case RDMA_AH_ATTR_TYPE_ROCE: > + case RDMA_AH_ATTR_TYPE_UNDEFINED: > + *dlid = 0; > + break; > + } > +} > +#define CREATE_TRACE_POINTS > +#include <trace/events/ib_mad.h> > + > static int mad_sendq_size = IB_MAD_QP_SEND_SIZE; > static int mad_recvq_size = IB_MAD_QP_RECV_SIZE; > > @@ -1223,6 +1258,7 @@ int ib_send_mad(struct ib_mad_send_wr_private > *mad_send_wr) > spin_lock_irqsave(&qp_info->send_queue.lock, flags); > if (qp_info->send_queue.count < > qp_info->send_queue.max_active) { > + trace_ib_mad_ib_send_mad(mad_send_wr, qp_info); > ret = ib_post_send(mad_agent->qp, > &mad_send_wr->send_wr.wr, NULL); > list = &qp_info->send_queue.list; > @@ -2496,6 +2532,8 @@ static void ib_mad_send_done(struct ib_cq *cq, > struct ib_wc *wc) send_queue = mad_list->mad_queue; > qp_info = send_queue->qp_info; > > + trace_ib_mad_send_done_handler(mad_send_wr, wc); > + > retry: > ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device, > mad_send_wr->header_mapping, > @@ -2527,6 +2565,7 @@ static void ib_mad_send_done(struct ib_cq *cq, > struct ib_wc *wc) ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc); > > if (queued_send_wr) { > + trace_ib_mad_send_done_resend(queued_send_wr, > qp_info); ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr, > NULL); > if (ret) { > @@ -2574,6 +2613,7 @@ static bool ib_mad_send_error(struct > ib_mad_port_private *port_priv, if (mad_send_wr->retry) { > /* Repost send */ > mad_send_wr->retry = 0; > + trace_ib_mad_error_handler(mad_send_wr, > qp_info); ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr, > NULL); > if (!ret) > diff --git a/include/trace/events/ib_mad.h > b/include/trace/events/ib_mad.h new file mode 100644 > index 000000000000..09c72ca1e857 > --- /dev/null > +++ b/include/trace/events/ib_mad.h > @@ -0,0 +1,183 @@ > +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ > + > +/* > + * Copyright (c) 2018 Intel Corporation. All rights reserved. > + */ > + > +#undef TRACE_SYSTEM > +#define TRACE_SYSTEM ib_mad > + > +#if !defined(_TRACE_IB_MAD_H) || defined(TRACE_HEADER_MULTI_READ) > +#define _TRACE_IB_MAD_H > + > +#include <linux/tracepoint.h> > +#include <rdma/ib_mad.h> > + > +DECLARE_EVENT_CLASS(ib_mad_send_template, > + TP_PROTO(struct ib_mad_send_wr_private *wr, > + struct ib_mad_qp_info *qp_info), > + TP_ARGS(wr, qp_info), > + > + TP_STRUCT__entry( > + __field(u8, base_version) > + __field(u8, mgmt_class) > + __field(u8, class_version) > + __field(u8, port_num) > + __field(u32, qp_num) > + __field(u8, method) > + __field(u8, sl) > + __field(u16, attr_id) > + __field(u32, attr_mod) > + __field(u64, wrtid) > + __field(u64, tid) > + __field(u16, status) > + __field(u16, class_specific) > + __field(u32, length) > + __field(u32, dlid) > + __field(u32, rqpn) > + __field(u32, rqkey) > + __field(u32, dev_index) > + __field(void *, agent_priv) > + __field(unsigned long, timeout) > + __field(int, retries_left) > + __field(int, max_retries) > + __field(int, retry) > + __field(u16, pkey) > + ), > + > + TP_fast_assign( > + __entry->dev_index = > wr->mad_agent_priv->agent.device->index; > + __entry->port_num = > wr->mad_agent_priv->agent.port_num; > + __entry->qp_num = > wr->mad_agent_priv->qp_info->qp->qp_num; > + __entry->agent_priv = wr->mad_agent_priv; > + __entry->wrtid = wr->tid; > + __entry->max_retries = wr->max_retries; > + __entry->retries_left = wr->retries_left; > + __entry->retry = wr->retry; > + __entry->timeout = wr->timeout; > + __entry->length = wr->send_buf.hdr_len + > + wr->send_buf.data_len; > + __entry->base_version = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->base_version; > + __entry->mgmt_class = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->mgmt_class; > + __entry->class_version = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->class_version; > + __entry->method = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->method; > + __entry->status = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->status; > + __entry->class_specific = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->class_specific; > + __entry->tid = ((struct ib_mad_hdr > *)wr->send_buf.mad)->tid; > + __entry->attr_id = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->attr_id; > + __entry->attr_mod = > + ((struct ib_mad_hdr > *)wr->send_buf.mad)->attr_mod; > + create_mad_addr_info(wr, qp_info, > + &__entry->dlid, &__entry->sl, > + &__entry->pkey, &__entry->rqpn, > + &__entry->rqkey); > + ), > + > + TP_printk("%d:%d QP%d agent %p: " \ > + "wrtid 0x%llx; %d/%d retries(%d); timeout %lu > length %d : " \ > + "hdr : base_ver 0x%x class 0x%x class_ver 0x%x " \ > + "method 0x%x status 0x%x class_specific 0x%x tid > 0x%llx " \ > + "attr_id 0x%x attr_mod 0x%x => dlid 0x%08x sl %d > "\ > + "pkey 0x%x rpqn 0x%x rqpkey 0x%x", > + __entry->dev_index, __entry->port_num, > __entry->qp_num, > + __entry->agent_priv, be64_to_cpu(__entry->wrtid), > + __entry->retries_left, __entry->max_retries, > + __entry->retry, __entry->timeout, __entry->length, > + __entry->base_version, __entry->mgmt_class, > + __entry->class_version, > + __entry->method, be16_to_cpu(__entry->status), > + be16_to_cpu(__entry->class_specific), > + be64_to_cpu(__entry->tid), > be16_to_cpu(__entry->attr_id), > + be32_to_cpu(__entry->attr_mod), > + be32_to_cpu(__entry->dlid), __entry->sl, > __entry->pkey, > + __entry->rqpn, __entry->rqkey > + ) > +); > + > +DEFINE_EVENT(ib_mad_send_template, ib_mad_error_handler, > + TP_PROTO(struct ib_mad_send_wr_private *wr, > + struct ib_mad_qp_info *qp_info), > + TP_ARGS(wr, qp_info)); > +DEFINE_EVENT(ib_mad_send_template, ib_mad_ib_send_mad, > + TP_PROTO(struct ib_mad_send_wr_private *wr, > + struct ib_mad_qp_info *qp_info), > + TP_ARGS(wr, qp_info)); > +DEFINE_EVENT(ib_mad_send_template, ib_mad_send_done_resend, > + TP_PROTO(struct ib_mad_send_wr_private *wr, > + struct ib_mad_qp_info *qp_info), > + TP_ARGS(wr, qp_info)); > +
On Mon, Feb 25, 2019 at 11:36:40AM -0500, Steven Rostedt wrote: > On Sun, 24 Feb 2019 21:11:59 -0800 > ira.weiny@intel.com wrote: > > > INGENIC JZ4780 DMA Driver > > M: Zubair Lutfullah Kakakhel <Zubair.Kakakhel@imgtec.com> > > diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c > > index e742a6a2c138..d5cd90fecda7 100644 > > --- a/drivers/infiniband/core/mad.c > > +++ b/drivers/infiniband/core/mad.c > > @@ -3,7 +3,7 @@ > > * Copyright (c) 2005 Intel Corporation. All rights reserved. > > * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. > > * Copyright (c) 2009 HNR Consulting. All rights reserved. > > - * Copyright (c) 2014 Intel Corporation. All rights reserved. > > + * Copyright (c) 2014,2018 Intel Corporation. All rights reserved. > > * > > * This software is available to you under a choice of one of two > > * licenses. You may choose to be licensed under the terms of the GNU > > @@ -51,6 +51,41 @@ > > #include "opa_smi.h" > > #include "agent.h" > > > > +static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, > > + struct ib_mad_qp_info *qp_info, > > + u32 *dlid, u8 *sl, u16 *pkey, u32 *rqpn, > > + u32 *rqkey) > > +{ > > Instead of passing in a bunch of addresses, you may be able to pass in > the entry structure directly. > > struct trace_event_raw_ib_mad_send_template *entry > > (defined from DECLARE_EVENT_CLASS(name, ... > > struct trace_event_raw_##name .. ) > > And just assign everything as: > > entry->dlid = x, ... raw_tracepoint requires all args to fit into u64. bpf prog cannot access it otherwise. Also struct by value is pretty slow on some archs. Pointer to struct is always preferred.
On Mon, 25 Feb 2019 21:52:46 -0800 Alexei Starovoitov <alexei.starovoitov@gmail.com> wrote: > On Mon, Feb 25, 2019 at 11:36:40AM -0500, Steven Rostedt wrote: > > On Sun, 24 Feb 2019 21:11:59 -0800 > > ira.weiny@intel.com wrote: > > > > > INGENIC JZ4780 DMA Driver > > > M: Zubair Lutfullah Kakakhel <Zubair.Kakakhel@imgtec.com> > > > diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c > > > index e742a6a2c138..d5cd90fecda7 100644 > > > --- a/drivers/infiniband/core/mad.c > > > +++ b/drivers/infiniband/core/mad.c > > > @@ -3,7 +3,7 @@ > > > * Copyright (c) 2005 Intel Corporation. All rights reserved. > > > * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. > > > * Copyright (c) 2009 HNR Consulting. All rights reserved. > > > - * Copyright (c) 2014 Intel Corporation. All rights reserved. > > > + * Copyright (c) 2014,2018 Intel Corporation. All rights reserved. > > > * > > > * This software is available to you under a choice of one of two > > > * licenses. You may choose to be licensed under the terms of the GNU > > > @@ -51,6 +51,41 @@ > > > #include "opa_smi.h" > > > #include "agent.h" > > > > > > +static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, > > > + struct ib_mad_qp_info *qp_info, > > > + u32 *dlid, u8 *sl, u16 *pkey, u32 *rqpn, > > > + u32 *rqkey) > > > +{ > > > > Instead of passing in a bunch of addresses, you may be able to pass in > > the entry structure directly. > > > > struct trace_event_raw_ib_mad_send_template *entry > > > > (defined from DECLARE_EVENT_CLASS(name, ... > > > > struct trace_event_raw_##name .. ) > > > > And just assign everything as: > > > > entry->dlid = x, ... > > raw_tracepoint requires all args to fit into u64. > bpf prog cannot access it otherwise. > Also struct by value is pretty slow on some archs. > Pointer to struct is always preferred. I meant passing in the pointer, not the value. It wouldn't make sense otherwise, because the entire point of create_mad_addr_info(), is to update the __entry structure. How would passing in the value do that? Instead of: + create_mad_addr_info(wr, qp_info, + &__entry->dlid, &__entry->sl, + &__entry->pkey, &__entry->rqpn, + &__entry->rqkey); You do: create_mad_addr_info(wr, qp_info, __entry); And then in that function you assign the values directly to the structure. -- Steve
On Tue, Feb 26, 2019 at 01:13:17AM -0500, Steven Rostedt wrote: > On Mon, 25 Feb 2019 21:52:46 -0800 > Alexei Starovoitov <alexei.starovoitov@gmail.com> wrote: > > > On Mon, Feb 25, 2019 at 11:36:40AM -0500, Steven Rostedt wrote: > > > On Sun, 24 Feb 2019 21:11:59 -0800 > > > ira.weiny@intel.com wrote: > > > > > > > > +static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, > > > > + struct ib_mad_qp_info *qp_info, > > > > + u32 *dlid, u8 *sl, u16 *pkey, u32 *rqpn, > > > > + u32 *rqkey) > > > > +{ > > > > > > Instead of passing in a bunch of addresses, you may be able to pass in > > > the entry structure directly. > > > > > > struct trace_event_raw_ib_mad_send_template *entry > > > > > > (defined from DECLARE_EVENT_CLASS(name, ... > > > > > > struct trace_event_raw_##name .. ) > > > > > > And just assign everything as: > > > > > > entry->dlid = x, ... > > > > raw_tracepoint requires all args to fit into u64. > > bpf prog cannot access it otherwise. > > Also struct by value is pretty slow on some archs. > > Pointer to struct is always preferred. > > I meant passing in the pointer, not the value. It wouldn't make sense > otherwise, because the entire point of create_mad_addr_info(), is to > update the __entry structure. How would passing in the value do that? > > > Instead of: > > + create_mad_addr_info(wr, qp_info, > + &__entry->dlid, &__entry->sl, > + &__entry->pkey, &__entry->rqpn, > + &__entry->rqkey); > > You do: > > create_mad_addr_info(wr, qp_info, __entry); > > And then in that function you assign the values directly to the structure. Is there any reason this is preferred over what I did? I'm not seeing the benefit? Ira > > -- Steve > >
On Tue, 26 Feb 2019 08:19:56 -0800 Ira Weiny <ira.weiny@intel.com> wrote: > > Instead of: > > > > + create_mad_addr_info(wr, qp_info, > > + &__entry->dlid, &__entry->sl, > > + &__entry->pkey, &__entry->rqpn, > > + &__entry->rqkey); > > > > You do: > > > > create_mad_addr_info(wr, qp_info, __entry); > > > > And then in that function you assign the values directly to the structure. > > Is there any reason this is preferred over what I did? I'm not seeing the > benefit? > Matters if gcc optimizes it into a static inline (which you may not want, because that goes directly into the code that calls the trace points, for each event in that class. Thus, I believe it is a true function call. In which, it is passing 7 arguments, on x86, that requires storing on the stack, and takes time to set up and tear down. Passing the function pointer directly, makes it only use 3 arguments (all in registers) and is much faster, especially, if this is a fast path). Do a disassemble of both approaches to see what I mean. -- Steve
On Tue, Feb 26, 2019 at 01:33:30PM -0500, Steven Rostedt wrote: > On Tue, 26 Feb 2019 08:19:56 -0800 > Ira Weiny <ira.weiny@intel.com> wrote: > > > > > Instead of: > > > > > > + create_mad_addr_info(wr, qp_info, > > > + &__entry->dlid, &__entry->sl, > > > + &__entry->pkey, &__entry->rqpn, > > > + &__entry->rqkey); > > > > > > You do: > > > > > > create_mad_addr_info(wr, qp_info, __entry); > > > > > > And then in that function you assign the values directly to the structure. > > > > Is there any reason this is preferred over what I did? I'm not seeing the > > benefit? > > > > Matters if gcc optimizes it into a static inline (which you may not > want, because that goes directly into the code that calls the trace > points, for each event in that class. Thus, I believe it is a true > function call. In which, it is passing 7 arguments, on x86, that > requires storing on the stack, and takes time to set up and tear down. > > Passing the function pointer directly, makes it only use 3 arguments > (all in registers) and is much faster, especially, if this is a fast > path). > > Do a disassemble of both approaches to see what I mean. Right... However, this results in quite the chicken and the egg problem because calling create_mad_addr_info() within the definition of trace_event_raw_ib_mad_send_template results in errors that I'm declaring the structure in the parameter list... :-/ I've tried moving create_mad_addr_info() around a bit and a couple of things with void pointers but the best I can get is that I'm dereferencing an incomplete type because the output of the preprocessor always seems to put at least 1 call of create_mad_addr_info() prior to the struct definition. Is there an example of this somewhere? This code is not that performance critical but I am curious at this point. Ira > > -- Steve
On Tue, 26 Feb 2019 18:51:37 -0800 Ira Weiny <ira.weiny@intel.com> wrote: > > Do a disassemble of both approaches to see what I mean. > > Right... However, this results in quite the chicken and the egg problem > because calling create_mad_addr_info() within the definition of > trace_event_raw_ib_mad_send_template results in errors that I'm > declaring the structure in the parameter list... Just adding: struct trace_event_raw_ib_mad_send_template; Before the macro should fix it. > > :-/ > > I've tried moving create_mad_addr_info() around a bit and a couple of things > with void pointers but the best I can get is that I'm dereferencing an > incomplete type because the output of the preprocessor always seems to put at > least 1 call of create_mad_addr_info() prior to the struct definition. > > Is there an example of this somewhere? This code is not that performance > critical but I am curious at this point. Not really, this is a unique case where you have a single function updating several parameters. -- Steve
On Wed, Feb 27, 2019 at 09:10:05PM -0500, Steven Rostedt wrote: > On Tue, 26 Feb 2019 18:51:37 -0800 > Ira Weiny <ira.weiny@intel.com> wrote: > > > > > Do a disassemble of both approaches to see what I mean. > > > > Right... However, this results in quite the chicken and the egg problem > > because calling create_mad_addr_info() within the definition of > > trace_event_raw_ib_mad_send_template results in errors that I'm > > declaring the structure in the parameter list... > > Just adding: > > struct trace_event_raw_ib_mad_send_template; > > Before the macro should fix it. Ok the key was to pre-declare both the struct and create_mad_addr_info() as shown below. Jason I'm going to send this as a follow on to the series. Ira From 5fcbff09a0b076c165dd3f9fd65df7558df9c58b Mon Sep 17 00:00:00 2001 From: Ira Weiny <ira.weiny@intel.com> Date: Wed, 27 Feb 2019 19:16:35 -0500 Subject: [PATCH] IB/mad: Use trace struct in create_mad_addr_info() helper Using a structure pointer is more efficient in the helper. Signed-off-by: Ira Weiny <ira.weiny@intel.com> --- drivers/infiniband/core/mad.c | 26 ++++++++++++++------------ include/trace/events/ib_mad.h | 10 ++++++---- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index 87eff3f3f887..0a6136dded47 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -51,11 +51,14 @@ #include "opa_smi.h" #include "agent.h" +#define CREATE_TRACE_POINTS +#include <trace/events/ib_mad.h> + static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, - struct ib_mad_qp_info *qp_info, - u32 *dlid, u8 *sl, u16 *pkey, u32 *rqpn, - u32 *rqkey) + struct ib_mad_qp_info *qp_info, + struct trace_event_raw_ib_mad_send_template *entry) { + u16 pkey; struct ib_device *dev = qp_info->port_priv->device; u8 pnum = qp_info->port_priv->port_num; struct ib_ud_wr *wr = &mad_send_wr->send_wr; @@ -65,26 +68,25 @@ static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, rdma_query_ah(wr->ah, &attr); /* These are common */ - *sl = attr.sl; - ib_query_pkey(dev, pnum, wr->pkey_index, pkey); - *rqpn = wr->remote_qpn; - *rqkey = wr->remote_qkey; + entry->sl = attr.sl; + ib_query_pkey(dev, pnum, wr->pkey_index, &pkey); + entry->pkey = pkey; + entry->rqpn = wr->remote_qpn; + entry->rqkey = wr->remote_qkey; switch (attr.type) { case RDMA_AH_ATTR_TYPE_IB: - *dlid = attr.ib.dlid; + entry->dlid = attr.ib.dlid; break; case RDMA_AH_ATTR_TYPE_OPA: - *dlid = attr.opa.dlid; + entry->dlid = attr.opa.dlid; break; case RDMA_AH_ATTR_TYPE_ROCE: case RDMA_AH_ATTR_TYPE_UNDEFINED: - *dlid = 0; + entry->dlid = 0; break; } } -#define CREATE_TRACE_POINTS -#include <trace/events/ib_mad.h> static int mad_sendq_size = IB_MAD_QP_SEND_SIZE; static int mad_recvq_size = IB_MAD_QP_RECV_SIZE; diff --git a/include/trace/events/ib_mad.h b/include/trace/events/ib_mad.h index 7f2486535d04..cccb2cf4a8f7 100644 --- a/include/trace/events/ib_mad.h +++ b/include/trace/events/ib_mad.h @@ -13,6 +13,11 @@ #include <linux/tracepoint.h> #include <rdma/ib_mad.h> +struct trace_event_raw_ib_mad_send_template; +static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, + struct ib_mad_qp_info *qp_info, + struct trace_event_raw_ib_mad_send_template *entry); + DECLARE_EVENT_CLASS(ib_mad_send_template, TP_PROTO(struct ib_mad_send_wr_private *wr, struct ib_mad_qp_info *qp_info), @@ -74,10 +79,7 @@ DECLARE_EVENT_CLASS(ib_mad_send_template, ((struct ib_mad_hdr *)wr->send_buf.mad)->attr_id; __entry->attr_mod = ((struct ib_mad_hdr *)wr->send_buf.mad)->attr_mod; - create_mad_addr_info(wr, qp_info, - &__entry->dlid, &__entry->sl, - &__entry->pkey, &__entry->rqpn, - &__entry->rqkey); + create_mad_addr_info(wr, qp_info, __entry); ), TP_printk("%d:%d QP%d agent %p: " \
diff --git a/MAINTAINERS b/MAINTAINERS index 459bed632ebb..6ad8f3f7a580 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7506,6 +7506,7 @@ F: drivers/infiniband/ F: include/uapi/linux/if_infiniband.h F: include/uapi/rdma/ F: include/rdma/ +F: include/trace/events/ib_mad.h INGENIC JZ4780 DMA Driver M: Zubair Lutfullah Kakakhel <Zubair.Kakakhel@imgtec.com> diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index e742a6a2c138..d5cd90fecda7 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -3,7 +3,7 @@ * Copyright (c) 2005 Intel Corporation. All rights reserved. * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. - * Copyright (c) 2014 Intel Corporation. All rights reserved. + * Copyright (c) 2014,2018 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -51,6 +51,41 @@ #include "opa_smi.h" #include "agent.h" +static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, + struct ib_mad_qp_info *qp_info, + u32 *dlid, u8 *sl, u16 *pkey, u32 *rqpn, + u32 *rqkey) +{ + struct ib_device *dev = qp_info->port_priv->device; + u8 pnum = qp_info->port_priv->port_num; + struct ib_ud_wr *wr = &mad_send_wr->send_wr; + struct rdma_ah_attr attr; + + memset(&attr, 0, sizeof(attr)); + rdma_query_ah(wr->ah, &attr); + + /* These are common */ + *sl = attr.sl; + ib_query_pkey(dev, pnum, wr->pkey_index, pkey); + *rqpn = wr->remote_qpn; + *rqkey = wr->remote_qkey; + + switch (attr.type) { + case RDMA_AH_ATTR_TYPE_IB: + *dlid = attr.ib.dlid; + break; + case RDMA_AH_ATTR_TYPE_OPA: + *dlid = attr.opa.dlid; + break; + case RDMA_AH_ATTR_TYPE_ROCE: + case RDMA_AH_ATTR_TYPE_UNDEFINED: + *dlid = 0; + break; + } +} +#define CREATE_TRACE_POINTS +#include <trace/events/ib_mad.h> + static int mad_sendq_size = IB_MAD_QP_SEND_SIZE; static int mad_recvq_size = IB_MAD_QP_RECV_SIZE; @@ -1223,6 +1258,7 @@ int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr) spin_lock_irqsave(&qp_info->send_queue.lock, flags); if (qp_info->send_queue.count < qp_info->send_queue.max_active) { + trace_ib_mad_ib_send_mad(mad_send_wr, qp_info); ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr, NULL); list = &qp_info->send_queue.list; @@ -2496,6 +2532,8 @@ static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc) send_queue = mad_list->mad_queue; qp_info = send_queue->qp_info; + trace_ib_mad_send_done_handler(mad_send_wr, wc); + retry: ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device, mad_send_wr->header_mapping, @@ -2527,6 +2565,7 @@ static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc) ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc); if (queued_send_wr) { + trace_ib_mad_send_done_resend(queued_send_wr, qp_info); ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr, NULL); if (ret) { @@ -2574,6 +2613,7 @@ static bool ib_mad_send_error(struct ib_mad_port_private *port_priv, if (mad_send_wr->retry) { /* Repost send */ mad_send_wr->retry = 0; + trace_ib_mad_error_handler(mad_send_wr, qp_info); ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr, NULL); if (!ret) diff --git a/include/trace/events/ib_mad.h b/include/trace/events/ib_mad.h new file mode 100644 index 000000000000..09c72ca1e857 --- /dev/null +++ b/include/trace/events/ib_mad.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ + +/* + * Copyright (c) 2018 Intel Corporation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ib_mad + +#if !defined(_TRACE_IB_MAD_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_IB_MAD_H + +#include <linux/tracepoint.h> +#include <rdma/ib_mad.h> + +DECLARE_EVENT_CLASS(ib_mad_send_template, + TP_PROTO(struct ib_mad_send_wr_private *wr, + struct ib_mad_qp_info *qp_info), + TP_ARGS(wr, qp_info), + + TP_STRUCT__entry( + __field(u8, base_version) + __field(u8, mgmt_class) + __field(u8, class_version) + __field(u8, port_num) + __field(u32, qp_num) + __field(u8, method) + __field(u8, sl) + __field(u16, attr_id) + __field(u32, attr_mod) + __field(u64, wrtid) + __field(u64, tid) + __field(u16, status) + __field(u16, class_specific) + __field(u32, length) + __field(u32, dlid) + __field(u32, rqpn) + __field(u32, rqkey) + __field(u32, dev_index) + __field(void *, agent_priv) + __field(unsigned long, timeout) + __field(int, retries_left) + __field(int, max_retries) + __field(int, retry) + __field(u16, pkey) + ), + + TP_fast_assign( + __entry->dev_index = wr->mad_agent_priv->agent.device->index; + __entry->port_num = wr->mad_agent_priv->agent.port_num; + __entry->qp_num = wr->mad_agent_priv->qp_info->qp->qp_num; + __entry->agent_priv = wr->mad_agent_priv; + __entry->wrtid = wr->tid; + __entry->max_retries = wr->max_retries; + __entry->retries_left = wr->retries_left; + __entry->retry = wr->retry; + __entry->timeout = wr->timeout; + __entry->length = wr->send_buf.hdr_len + + wr->send_buf.data_len; + __entry->base_version = + ((struct ib_mad_hdr *)wr->send_buf.mad)->base_version; + __entry->mgmt_class = + ((struct ib_mad_hdr *)wr->send_buf.mad)->mgmt_class; + __entry->class_version = + ((struct ib_mad_hdr *)wr->send_buf.mad)->class_version; + __entry->method = + ((struct ib_mad_hdr *)wr->send_buf.mad)->method; + __entry->status = + ((struct ib_mad_hdr *)wr->send_buf.mad)->status; + __entry->class_specific = + ((struct ib_mad_hdr *)wr->send_buf.mad)->class_specific; + __entry->tid = ((struct ib_mad_hdr *)wr->send_buf.mad)->tid; + __entry->attr_id = + ((struct ib_mad_hdr *)wr->send_buf.mad)->attr_id; + __entry->attr_mod = + ((struct ib_mad_hdr *)wr->send_buf.mad)->attr_mod; + create_mad_addr_info(wr, qp_info, + &__entry->dlid, &__entry->sl, + &__entry->pkey, &__entry->rqpn, + &__entry->rqkey); + ), + + TP_printk("%d:%d QP%d agent %p: " \ + "wrtid 0x%llx; %d/%d retries(%d); timeout %lu length %d : " \ + "hdr : base_ver 0x%x class 0x%x class_ver 0x%x " \ + "method 0x%x status 0x%x class_specific 0x%x tid 0x%llx " \ + "attr_id 0x%x attr_mod 0x%x => dlid 0x%08x sl %d "\ + "pkey 0x%x rpqn 0x%x rqpkey 0x%x", + __entry->dev_index, __entry->port_num, __entry->qp_num, + __entry->agent_priv, be64_to_cpu(__entry->wrtid), + __entry->retries_left, __entry->max_retries, + __entry->retry, __entry->timeout, __entry->length, + __entry->base_version, __entry->mgmt_class, + __entry->class_version, + __entry->method, be16_to_cpu(__entry->status), + be16_to_cpu(__entry->class_specific), + be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id), + be32_to_cpu(__entry->attr_mod), + be32_to_cpu(__entry->dlid), __entry->sl, __entry->pkey, + __entry->rqpn, __entry->rqkey + ) +); + +DEFINE_EVENT(ib_mad_send_template, ib_mad_error_handler, + TP_PROTO(struct ib_mad_send_wr_private *wr, + struct ib_mad_qp_info *qp_info), + TP_ARGS(wr, qp_info)); +DEFINE_EVENT(ib_mad_send_template, ib_mad_ib_send_mad, + TP_PROTO(struct ib_mad_send_wr_private *wr, + struct ib_mad_qp_info *qp_info), + TP_ARGS(wr, qp_info)); +DEFINE_EVENT(ib_mad_send_template, ib_mad_send_done_resend, + TP_PROTO(struct ib_mad_send_wr_private *wr, + struct ib_mad_qp_info *qp_info), + TP_ARGS(wr, qp_info)); + +TRACE_EVENT(ib_mad_send_done_handler, + TP_PROTO(struct ib_mad_send_wr_private *wr, struct ib_wc *wc), + TP_ARGS(wr, wc), + + TP_STRUCT__entry( + __field(u8, port_num) + __field(u8, base_version) + __field(u8, mgmt_class) + __field(u8, class_version) + __field(u32, qp_num) + __field(u64, wrtid) + __field(u16, status) + __field(u16, wc_status) + __field(u32, length) + __field(void *, agent_priv) + __field(unsigned long, timeout) + __field(u32, dev_index) + __field(int, retries_left) + __field(int, max_retries) + __field(int, retry) + __field(u8, method) + ), + + TP_fast_assign( + __entry->dev_index = wr->mad_agent_priv->agent.device->index; + __entry->port_num = wr->mad_agent_priv->agent.port_num; + __entry->qp_num = wr->mad_agent_priv->qp_info->qp->qp_num; + __entry->agent_priv = wr->mad_agent_priv; + __entry->wrtid = wr->tid; + __entry->max_retries = wr->max_retries; + __entry->retries_left = wr->retries_left; + __entry->retry = wr->retry; + __entry->timeout = wr->timeout; + __entry->base_version = + ((struct ib_mad_hdr *)wr->send_buf.mad)->base_version; + __entry->mgmt_class = + ((struct ib_mad_hdr *)wr->send_buf.mad)->mgmt_class; + __entry->class_version = + ((struct ib_mad_hdr *)wr->send_buf.mad)->class_version; + __entry->method = + ((struct ib_mad_hdr *)wr->send_buf.mad)->method; + __entry->status = + ((struct ib_mad_hdr *)wr->send_buf.mad)->status; + __entry->wc_status = wc->status; + __entry->length = wc->byte_len; + ), + + TP_printk("%d:%d QP%d : SEND WC Status %d : agent %p: " \ + "wrtid 0x%llx %d/%d retries(%d) timeout %lu length %d: " \ + "hdr : base_ver 0x%x class 0x%x class_ver 0x%x " \ + "method 0x%x status 0x%x", + __entry->dev_index, __entry->port_num, __entry->qp_num, + __entry->wc_status, + __entry->agent_priv, be64_to_cpu(__entry->wrtid), + __entry->retries_left, __entry->max_retries, + __entry->retry, __entry->timeout, + __entry->length, + __entry->base_version, __entry->mgmt_class, + __entry->class_version, __entry->method, + be16_to_cpu(__entry->status) + ) +); + + +#endif /* _TRACE_IB_MAD_H */ + +#include <trace/define_trace.h>