diff mbox

[v2,06/11] IB/mad: Ensure DR MADs are correctly specified when using OPA devices

Message ID 1479843532-47496-7-git-send-email-dasaratharaman.chandramouli@intel.com (mailing list archive)
State Changes Requested
Headers show

Commit Message

Dasaratharaman Chandramouli Nov. 22, 2016, 7:38 p.m. UTC
From: Don Hiatt <don.hiatt@intel.com>

Pure DR MADs do not need OPA GIDs to be specified in the GRH since
they do not rely on LID information.

Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
Signed-off-by: Don Hiatt <don.hiatt@intel.com>
---
 drivers/infiniband/core/mad.c | 104 +++++++++++++++++++++++++++++++++++++-----
 include/rdma/opa_addr.h       |  17 +++++++
 2 files changed, 109 insertions(+), 12 deletions(-)

Comments

Hal Rosenstock Nov. 23, 2016, 2:21 p.m. UTC | #1
On 11/22/2016 2:38 PM, Dasaratharaman Chandramouli wrote:
> From: Don Hiatt <don.hiatt@intel.com>
> 
> Pure DR MADs do not need OPA GIDs to be specified in the GRH since
> they do not rely on LID information.
> 
> Reviewed-by: Ira Weiny <ira.weiny@intel.com>
> Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
> Signed-off-by: Don Hiatt <don.hiatt@intel.com>
> ---
>  drivers/infiniband/core/mad.c | 104 +++++++++++++++++++++++++++++++++++++-----
>  include/rdma/opa_addr.h       |  17 +++++++
>  2 files changed, 109 insertions(+), 12 deletions(-)
> 
> diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
> index 40cbd6b..c0ee997 100644
> --- a/drivers/infiniband/core/mad.c
> +++ b/drivers/infiniband/core/mad.c
> @@ -41,6 +41,7 @@
>  #include <linux/slab.h>
>  #include <linux/module.h>
>  #include <rdma/ib_cache.h>
> +#include <rdma/opa_addr.h>
>  
>  #include "mad_priv.h"
>  #include "mad_rmpp.h"
> @@ -731,6 +732,80 @@ static size_t mad_priv_dma_size(const struct ib_mad_private *mp)
>  	return sizeof(struct ib_grh) + mp->mad_size;
>  }
>  
> +static int verify_mad_ah(struct ib_mad_agent_private *mad_agent_priv,

I think it would be better if this were named opa_verify_mad_ah to make
it clearer that this is an OPA only routine.

-- Hal

> +			 struct ib_mad_send_wr_private *mad_send_wr)
> +{
> +	struct ib_device *ib_dev = mad_agent_priv->qp_info->port_priv->device;
> +	u8 port = mad_agent_priv->qp_info->port_priv->port_num;
> +	struct ib_smp *smp = mad_send_wr->send_buf.mad;
> +	struct opa_smp *opa_smp = (struct opa_smp *)smp;
> +	u32 opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
> +	u32 opa_drdlid = be32_to_cpu(opa_smp->route.dr.dr_dlid);
> +
> +	bool dr_slid_is_permissive = (OPA_LID_PERMISSIVE ==
> +				      opa_smp->route.dr.dr_slid) ? true : false;
> +	bool dr_dlid_is_permissive = (OPA_LID_PERMISSIVE ==
> +				      opa_smp->route.dr.dr_dlid) ? true : false;
> +	bool drslid_is_ib_ucast = (opa_drslid <
> +				   be16_to_cpu(IB_MULTICAST_LID_BASE)) ?
> +					true : false;
> +	bool drdlid_is_ib_ucast = (opa_drdlid <
> +				   be16_to_cpu(IB_MULTICAST_LID_BASE)) ?
> +					true : false;
> +	bool drslid_is_ext = !drslid_is_ib_ucast && !dr_slid_is_permissive;
> +	bool drdlid_is_ext = !drdlid_is_ib_ucast && !dr_dlid_is_permissive;
> +	bool grh_present = false;
> +	struct ib_ah_attr attr;
> +	union ib_gid sgid;
> +	int ret = 0;
> +
> +	ret = ib_query_ah(mad_send_wr->send_buf.ah, &attr);
> +	if (ret)
> +		return ret;
> +	grh_present = (attr.ah_flags & IB_AH_GRH);
> +	if (grh_present) {
> +		ret = ib_query_gid(ib_dev, port, attr.grh.sgid_index,
> +				   &sgid, NULL);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	if (smp->class_version == OPA_SMP_CLASS_VERSION) {
> +		/*
> +		 * Conditions when GRH info should not be specified
> +		 * 1. both dr_slid and dr_dlid are permissve (Pure DR)
> +		 * 2. both dr_slid and dr_dlid are less than 0xc000.
> +		 *
> +		 * Conditions when GRH info should be specified
> +		 * 1. dr_dlid is not permissive and above 0xbfff
> +		 * OR
> +		 * 2. dr_slid is not permissive and above 0xbfff
> +		 */
> +		if (grh_present) {
> +			if ((dr_slid_is_permissive &&
> +			     dr_dlid_is_permissive) ||
> +			     (drslid_is_ib_ucast && drdlid_is_ib_ucast))
> +				if (ib_is_opa_gid(&attr.grh.dgid) &&
> +				    ib_is_opa_gid(&sgid))
> +					return -EINVAL;
> +			if (drslid_is_ext && !ib_is_opa_gid(&sgid))
> +				return -EINVAL;
> +			if (drdlid_is_ext &&
> +			    !ib_is_opa_gid(&attr.grh.dgid))
> +				return -EINVAL;
> +		} else { /* There is no GRH */
> +			if (drslid_is_ext || drdlid_is_ext)
> +				return -EINVAL;
> +		}
> +	} else {
> +		if (grh_present)
> +			if (ib_is_opa_gid(&attr.grh.dgid) &&
> +			    ib_is_opa_gid(&sgid))
> +				return -EINVAL;
> +	}
> +	return ret;
> +}
> +
>  /*
>   * Return 0 if SMP is to be sent
>   * Return 1 if SMP was consumed locally (whether or not solicited)
> @@ -754,8 +829,12 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>  	size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv);
>  	u16 out_mad_pkey_index = 0;
>  	u16 drslid;
> -	bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
> -				    mad_agent_priv->qp_info->port_priv->port_num);
> +	bool opa_mad =
> +		rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
> +				 mad_agent_priv->qp_info->port_priv->port_num);
> +	bool opa_ah =
> +		rdma_cap_opa_ah(mad_agent_priv->qp_info->port_priv->device,
> +				mad_agent_priv->qp_info->port_priv->port_num);
>  
>  	if (rdma_cap_ib_switch(device) &&
>  	    smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
> @@ -763,13 +842,21 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>  	else
>  		port_num = mad_agent_priv->agent.port_num;
>  
> +	if (opa_mad && opa_ah) {
> +		ret = verify_mad_ah(mad_agent_priv, mad_send_wr);
> +		if (ret) {
> +			dev_err(&device->dev,
> +				"Error verifying MAD format\n");
> +			goto out;
> +		}
> +	}
>  	/*
>  	 * Directed route handling starts if the initial LID routed part of
>  	 * a request or the ending LID routed part of a response is empty.
>  	 * If we are at the start of the LID routed part, don't update the
>  	 * hop_ptr or hop_cnt.  See section 14.2.2, Vol 1 IB spec.
>  	 */
> -	if (opa && smp->class_version == OPA_SMP_CLASS_VERSION) {
> +	if (opa_mad && smp->class_version == OPA_SMP_CLASS_VERSION) {
>  		u32 opa_drslid;
>  
>  		if ((opa_get_smp_direction(opa_smp)
> @@ -783,13 +870,6 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>  			goto out;
>  		}
>  		opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
> -		if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) &&
> -		    opa_drslid & 0xffff0000) {
> -			ret = -EINVAL;
> -			dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n",
> -			       opa_drslid);
> -			goto out;
> -		}
>  		drslid = (u16)(opa_drslid & 0x0000ffff);
>  
>  		/* Check to post send on QP or process locally */
> @@ -834,7 +914,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>  		     send_wr->pkey_index,
>  		     send_wr->port_num, &mad_wc);
>  
> -	if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) {
> +	if (opa_mad && smp->base_version == OPA_MGMT_BASE_VERSION) {
>  		mad_wc.byte_len = mad_send_wr->send_buf.hdr_len
>  					+ mad_send_wr->send_buf.data_len
>  					+ sizeof(struct ib_grh);
> @@ -891,7 +971,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>  	}
>  
>  	local->mad_send_wr = mad_send_wr;
> -	if (opa) {
> +	if (opa_mad) {
>  		local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;
>  		local->return_wc_byte_len = mad_size;
>  	}
> diff --git a/include/rdma/opa_addr.h b/include/rdma/opa_addr.h
> index 142b327..3e22937 100644
> --- a/include/rdma/opa_addr.h
> +++ b/include/rdma/opa_addr.h
> @@ -33,6 +33,23 @@
>  #if !defined(OPA_ADDR_H)
>  #define OPA_ADDR_H
>  
> +#include <rdma/ib_verbs.h>
> +
>  #define OPA_TO_IB_UCAST_LID(x)	(((x) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) \
>  				 ? 0 : x)
> +#define	OPA_SPECIAL_OUI		(0x00066AULL)
> +
> +/**
> + * ib_is_opa_gid: Returns true if the top 24 bits of the gid
> + * contains the OPA_STL_OUI identifier. This identifies that
> + * the provided gid is a special purpose GID meant to carry
> + * extended LID information.
> + *
> + * @gid: The Global identifier
> + */
> +static inline bool ib_is_opa_gid(union ib_gid *gid)
> +{
> +	return ((be64_to_cpu(gid->global.interface_id) >> 40) ==
> +		OPA_SPECIAL_OUI);
> +}
>  #endif /* OPA_ADDR_H */
> 
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Dasaratharaman Chandramouli Nov. 23, 2016, 6:29 p.m. UTC | #2
On 11/23/2016 6:21 AM, Hal Rosenstock wrote:
> On 11/22/2016 2:38 PM, Dasaratharaman Chandramouli wrote:
>> From: Don Hiatt <don.hiatt@intel.com>
>>
>> Pure DR MADs do not need OPA GIDs to be specified in the GRH since
>> they do not rely on LID information.
>>
>> Reviewed-by: Ira Weiny <ira.weiny@intel.com>
>> Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
>> Signed-off-by: Don Hiatt <don.hiatt@intel.com>
>> ---
>>  drivers/infiniband/core/mad.c | 104 +++++++++++++++++++++++++++++++++++++-----
>>  include/rdma/opa_addr.h       |  17 +++++++
>>  2 files changed, 109 insertions(+), 12 deletions(-)
>>
>> diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
>> index 40cbd6b..c0ee997 100644
>> --- a/drivers/infiniband/core/mad.c
>> +++ b/drivers/infiniband/core/mad.c
>> @@ -41,6 +41,7 @@
>>  #include <linux/slab.h>
>>  #include <linux/module.h>
>>  #include <rdma/ib_cache.h>
>> +#include <rdma/opa_addr.h>
>>
>>  #include "mad_priv.h"
>>  #include "mad_rmpp.h"
>> @@ -731,6 +732,80 @@ static size_t mad_priv_dma_size(const struct ib_mad_private *mp)
>>  	return sizeof(struct ib_grh) + mp->mad_size;
>>  }
>>
>> +static int verify_mad_ah(struct ib_mad_agent_private *mad_agent_priv,
>
> I think it would be better if this were named opa_verify_mad_ah to make
> it clearer that this is an OPA only routine.
>
> -- Hal
Thanks. Will rename this to opa_verify_mad_ah

-Dasa
>
>> +			 struct ib_mad_send_wr_private *mad_send_wr)
>> +{
>> +	struct ib_device *ib_dev = mad_agent_priv->qp_info->port_priv->device;
>> +	u8 port = mad_agent_priv->qp_info->port_priv->port_num;
>> +	struct ib_smp *smp = mad_send_wr->send_buf.mad;
>> +	struct opa_smp *opa_smp = (struct opa_smp *)smp;
>> +	u32 opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
>> +	u32 opa_drdlid = be32_to_cpu(opa_smp->route.dr.dr_dlid);
>> +
>> +	bool dr_slid_is_permissive = (OPA_LID_PERMISSIVE ==
>> +				      opa_smp->route.dr.dr_slid) ? true : false;
>> +	bool dr_dlid_is_permissive = (OPA_LID_PERMISSIVE ==
>> +				      opa_smp->route.dr.dr_dlid) ? true : false;
>> +	bool drslid_is_ib_ucast = (opa_drslid <
>> +				   be16_to_cpu(IB_MULTICAST_LID_BASE)) ?
>> +					true : false;
>> +	bool drdlid_is_ib_ucast = (opa_drdlid <
>> +				   be16_to_cpu(IB_MULTICAST_LID_BASE)) ?
>> +					true : false;
>> +	bool drslid_is_ext = !drslid_is_ib_ucast && !dr_slid_is_permissive;
>> +	bool drdlid_is_ext = !drdlid_is_ib_ucast && !dr_dlid_is_permissive;
>> +	bool grh_present = false;
>> +	struct ib_ah_attr attr;
>> +	union ib_gid sgid;
>> +	int ret = 0;
>> +
>> +	ret = ib_query_ah(mad_send_wr->send_buf.ah, &attr);
>> +	if (ret)
>> +		return ret;
>> +	grh_present = (attr.ah_flags & IB_AH_GRH);
>> +	if (grh_present) {
>> +		ret = ib_query_gid(ib_dev, port, attr.grh.sgid_index,
>> +				   &sgid, NULL);
>> +		if (ret)
>> +			return ret;
>> +	}
>> +
>> +	if (smp->class_version == OPA_SMP_CLASS_VERSION) {
>> +		/*
>> +		 * Conditions when GRH info should not be specified
>> +		 * 1. both dr_slid and dr_dlid are permissve (Pure DR)
>> +		 * 2. both dr_slid and dr_dlid are less than 0xc000.
>> +		 *
>> +		 * Conditions when GRH info should be specified
>> +		 * 1. dr_dlid is not permissive and above 0xbfff
>> +		 * OR
>> +		 * 2. dr_slid is not permissive and above 0xbfff
>> +		 */
>> +		if (grh_present) {
>> +			if ((dr_slid_is_permissive &&
>> +			     dr_dlid_is_permissive) ||
>> +			     (drslid_is_ib_ucast && drdlid_is_ib_ucast))
>> +				if (ib_is_opa_gid(&attr.grh.dgid) &&
>> +				    ib_is_opa_gid(&sgid))
>> +					return -EINVAL;
>> +			if (drslid_is_ext && !ib_is_opa_gid(&sgid))
>> +				return -EINVAL;
>> +			if (drdlid_is_ext &&
>> +			    !ib_is_opa_gid(&attr.grh.dgid))
>> +				return -EINVAL;
>> +		} else { /* There is no GRH */
>> +			if (drslid_is_ext || drdlid_is_ext)
>> +				return -EINVAL;
>> +		}
>> +	} else {
>> +		if (grh_present)
>> +			if (ib_is_opa_gid(&attr.grh.dgid) &&
>> +			    ib_is_opa_gid(&sgid))
>> +				return -EINVAL;
>> +	}
>> +	return ret;
>> +}
>> +
>>  /*
>>   * Return 0 if SMP is to be sent
>>   * Return 1 if SMP was consumed locally (whether or not solicited)
>> @@ -754,8 +829,12 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>>  	size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv);
>>  	u16 out_mad_pkey_index = 0;
>>  	u16 drslid;
>> -	bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
>> -				    mad_agent_priv->qp_info->port_priv->port_num);
>> +	bool opa_mad =
>> +		rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
>> +				 mad_agent_priv->qp_info->port_priv->port_num);
>> +	bool opa_ah =
>> +		rdma_cap_opa_ah(mad_agent_priv->qp_info->port_priv->device,
>> +				mad_agent_priv->qp_info->port_priv->port_num);
>>
>>  	if (rdma_cap_ib_switch(device) &&
>>  	    smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
>> @@ -763,13 +842,21 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>>  	else
>>  		port_num = mad_agent_priv->agent.port_num;
>>
>> +	if (opa_mad && opa_ah) {
>> +		ret = verify_mad_ah(mad_agent_priv, mad_send_wr);
>> +		if (ret) {
>> +			dev_err(&device->dev,
>> +				"Error verifying MAD format\n");
>> +			goto out;
>> +		}
>> +	}
>>  	/*
>>  	 * Directed route handling starts if the initial LID routed part of
>>  	 * a request or the ending LID routed part of a response is empty.
>>  	 * If we are at the start of the LID routed part, don't update the
>>  	 * hop_ptr or hop_cnt.  See section 14.2.2, Vol 1 IB spec.
>>  	 */
>> -	if (opa && smp->class_version == OPA_SMP_CLASS_VERSION) {
>> +	if (opa_mad && smp->class_version == OPA_SMP_CLASS_VERSION) {
>>  		u32 opa_drslid;
>>
>>  		if ((opa_get_smp_direction(opa_smp)
>> @@ -783,13 +870,6 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>>  			goto out;
>>  		}
>>  		opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
>> -		if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) &&
>> -		    opa_drslid & 0xffff0000) {
>> -			ret = -EINVAL;
>> -			dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n",
>> -			       opa_drslid);
>> -			goto out;
>> -		}
>>  		drslid = (u16)(opa_drslid & 0x0000ffff);
>>
>>  		/* Check to post send on QP or process locally */
>> @@ -834,7 +914,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>>  		     send_wr->pkey_index,
>>  		     send_wr->port_num, &mad_wc);
>>
>> -	if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) {
>> +	if (opa_mad && smp->base_version == OPA_MGMT_BASE_VERSION) {
>>  		mad_wc.byte_len = mad_send_wr->send_buf.hdr_len
>>  					+ mad_send_wr->send_buf.data_len
>>  					+ sizeof(struct ib_grh);
>> @@ -891,7 +971,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
>>  	}
>>
>>  	local->mad_send_wr = mad_send_wr;
>> -	if (opa) {
>> +	if (opa_mad) {
>>  		local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;
>>  		local->return_wc_byte_len = mad_size;
>>  	}
>> diff --git a/include/rdma/opa_addr.h b/include/rdma/opa_addr.h
>> index 142b327..3e22937 100644
>> --- a/include/rdma/opa_addr.h
>> +++ b/include/rdma/opa_addr.h
>> @@ -33,6 +33,23 @@
>>  #if !defined(OPA_ADDR_H)
>>  #define OPA_ADDR_H
>>
>> +#include <rdma/ib_verbs.h>
>> +
>>  #define OPA_TO_IB_UCAST_LID(x)	(((x) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) \
>>  				 ? 0 : x)
>> +#define	OPA_SPECIAL_OUI		(0x00066AULL)
>> +
>> +/**
>> + * ib_is_opa_gid: Returns true if the top 24 bits of the gid
>> + * contains the OPA_STL_OUI identifier. This identifies that
>> + * the provided gid is a special purpose GID meant to carry
>> + * extended LID information.
>> + *
>> + * @gid: The Global identifier
>> + */
>> +static inline bool ib_is_opa_gid(union ib_gid *gid)
>> +{
>> +	return ((be64_to_cpu(gid->global.interface_id) >> 40) ==
>> +		OPA_SPECIAL_OUI);
>> +}
>>  #endif /* OPA_ADDR_H */
>>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 40cbd6b..c0ee997 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -41,6 +41,7 @@ 
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <rdma/ib_cache.h>
+#include <rdma/opa_addr.h>
 
 #include "mad_priv.h"
 #include "mad_rmpp.h"
@@ -731,6 +732,80 @@  static size_t mad_priv_dma_size(const struct ib_mad_private *mp)
 	return sizeof(struct ib_grh) + mp->mad_size;
 }
 
+static int verify_mad_ah(struct ib_mad_agent_private *mad_agent_priv,
+			 struct ib_mad_send_wr_private *mad_send_wr)
+{
+	struct ib_device *ib_dev = mad_agent_priv->qp_info->port_priv->device;
+	u8 port = mad_agent_priv->qp_info->port_priv->port_num;
+	struct ib_smp *smp = mad_send_wr->send_buf.mad;
+	struct opa_smp *opa_smp = (struct opa_smp *)smp;
+	u32 opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
+	u32 opa_drdlid = be32_to_cpu(opa_smp->route.dr.dr_dlid);
+
+	bool dr_slid_is_permissive = (OPA_LID_PERMISSIVE ==
+				      opa_smp->route.dr.dr_slid) ? true : false;
+	bool dr_dlid_is_permissive = (OPA_LID_PERMISSIVE ==
+				      opa_smp->route.dr.dr_dlid) ? true : false;
+	bool drslid_is_ib_ucast = (opa_drslid <
+				   be16_to_cpu(IB_MULTICAST_LID_BASE)) ?
+					true : false;
+	bool drdlid_is_ib_ucast = (opa_drdlid <
+				   be16_to_cpu(IB_MULTICAST_LID_BASE)) ?
+					true : false;
+	bool drslid_is_ext = !drslid_is_ib_ucast && !dr_slid_is_permissive;
+	bool drdlid_is_ext = !drdlid_is_ib_ucast && !dr_dlid_is_permissive;
+	bool grh_present = false;
+	struct ib_ah_attr attr;
+	union ib_gid sgid;
+	int ret = 0;
+
+	ret = ib_query_ah(mad_send_wr->send_buf.ah, &attr);
+	if (ret)
+		return ret;
+	grh_present = (attr.ah_flags & IB_AH_GRH);
+	if (grh_present) {
+		ret = ib_query_gid(ib_dev, port, attr.grh.sgid_index,
+				   &sgid, NULL);
+		if (ret)
+			return ret;
+	}
+
+	if (smp->class_version == OPA_SMP_CLASS_VERSION) {
+		/*
+		 * Conditions when GRH info should not be specified
+		 * 1. both dr_slid and dr_dlid are permissve (Pure DR)
+		 * 2. both dr_slid and dr_dlid are less than 0xc000.
+		 *
+		 * Conditions when GRH info should be specified
+		 * 1. dr_dlid is not permissive and above 0xbfff
+		 * OR
+		 * 2. dr_slid is not permissive and above 0xbfff
+		 */
+		if (grh_present) {
+			if ((dr_slid_is_permissive &&
+			     dr_dlid_is_permissive) ||
+			     (drslid_is_ib_ucast && drdlid_is_ib_ucast))
+				if (ib_is_opa_gid(&attr.grh.dgid) &&
+				    ib_is_opa_gid(&sgid))
+					return -EINVAL;
+			if (drslid_is_ext && !ib_is_opa_gid(&sgid))
+				return -EINVAL;
+			if (drdlid_is_ext &&
+			    !ib_is_opa_gid(&attr.grh.dgid))
+				return -EINVAL;
+		} else { /* There is no GRH */
+			if (drslid_is_ext || drdlid_is_ext)
+				return -EINVAL;
+		}
+	} else {
+		if (grh_present)
+			if (ib_is_opa_gid(&attr.grh.dgid) &&
+			    ib_is_opa_gid(&sgid))
+				return -EINVAL;
+	}
+	return ret;
+}
+
 /*
  * Return 0 if SMP is to be sent
  * Return 1 if SMP was consumed locally (whether or not solicited)
@@ -754,8 +829,12 @@  static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv);
 	u16 out_mad_pkey_index = 0;
 	u16 drslid;
-	bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
-				    mad_agent_priv->qp_info->port_priv->port_num);
+	bool opa_mad =
+		rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
+				 mad_agent_priv->qp_info->port_priv->port_num);
+	bool opa_ah =
+		rdma_cap_opa_ah(mad_agent_priv->qp_info->port_priv->device,
+				mad_agent_priv->qp_info->port_priv->port_num);
 
 	if (rdma_cap_ib_switch(device) &&
 	    smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
@@ -763,13 +842,21 @@  static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	else
 		port_num = mad_agent_priv->agent.port_num;
 
+	if (opa_mad && opa_ah) {
+		ret = verify_mad_ah(mad_agent_priv, mad_send_wr);
+		if (ret) {
+			dev_err(&device->dev,
+				"Error verifying MAD format\n");
+			goto out;
+		}
+	}
 	/*
 	 * Directed route handling starts if the initial LID routed part of
 	 * a request or the ending LID routed part of a response is empty.
 	 * If we are at the start of the LID routed part, don't update the
 	 * hop_ptr or hop_cnt.  See section 14.2.2, Vol 1 IB spec.
 	 */
-	if (opa && smp->class_version == OPA_SMP_CLASS_VERSION) {
+	if (opa_mad && smp->class_version == OPA_SMP_CLASS_VERSION) {
 		u32 opa_drslid;
 
 		if ((opa_get_smp_direction(opa_smp)
@@ -783,13 +870,6 @@  static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 			goto out;
 		}
 		opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
-		if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) &&
-		    opa_drslid & 0xffff0000) {
-			ret = -EINVAL;
-			dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n",
-			       opa_drslid);
-			goto out;
-		}
 		drslid = (u16)(opa_drslid & 0x0000ffff);
 
 		/* Check to post send on QP or process locally */
@@ -834,7 +914,7 @@  static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 		     send_wr->pkey_index,
 		     send_wr->port_num, &mad_wc);
 
-	if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) {
+	if (opa_mad && smp->base_version == OPA_MGMT_BASE_VERSION) {
 		mad_wc.byte_len = mad_send_wr->send_buf.hdr_len
 					+ mad_send_wr->send_buf.data_len
 					+ sizeof(struct ib_grh);
@@ -891,7 +971,7 @@  static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	}
 
 	local->mad_send_wr = mad_send_wr;
-	if (opa) {
+	if (opa_mad) {
 		local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;
 		local->return_wc_byte_len = mad_size;
 	}
diff --git a/include/rdma/opa_addr.h b/include/rdma/opa_addr.h
index 142b327..3e22937 100644
--- a/include/rdma/opa_addr.h
+++ b/include/rdma/opa_addr.h
@@ -33,6 +33,23 @@ 
 #if !defined(OPA_ADDR_H)
 #define OPA_ADDR_H
 
+#include <rdma/ib_verbs.h>
+
 #define OPA_TO_IB_UCAST_LID(x)	(((x) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) \
 				 ? 0 : x)
+#define	OPA_SPECIAL_OUI		(0x00066AULL)
+
+/**
+ * ib_is_opa_gid: Returns true if the top 24 bits of the gid
+ * contains the OPA_STL_OUI identifier. This identifies that
+ * the provided gid is a special purpose GID meant to carry
+ * extended LID information.
+ *
+ * @gid: The Global identifier
+ */
+static inline bool ib_is_opa_gid(union ib_gid *gid)
+{
+	return ((be64_to_cpu(gid->global.interface_id) >> 40) ==
+		OPA_SPECIAL_OUI);
+}
 #endif /* OPA_ADDR_H */