[v2,3/4] common/multipath-over-rdma, tests/srp: Make it easy to use siw instead of rdma_rxe
diff mbox series

Message ID 20191213143232.29899-4-bvanassche@acm.org
State New
Headers show
Series
  • Add an SRP test for the SoftiWARP driver
Related show

Commit Message

Bart Van Assche Dec. 13, 2019, 2:32 p.m. UTC
Make all_primary_gids() return GIDs with zero suffix since SoftiWARP
GIDs have such a suffix. Add SoftiWARP support in start_soft_rdma() and
stop_soft_rdma(). Make do_rdma_cm_login() submit the source IP address
and scope ID to the SRP initiator driver since that information is
essential to establish an IPv6 connection between link-local addresses.
Iterate over RDMA ports instead of UMAD device nodes in log_in() since
no UMAD device node is associated with SoftiWARP ports. In start_lio_srpt(),
use the GID format to identify SoftiWARP ports since the GID suffix for
these ports is zero.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
 common/multipath-over-rdma | 48 ++++++++++++++++---
 tests/srp/rc               | 98 ++++++++++++++++++++++----------------
 2 files changed, 99 insertions(+), 47 deletions(-)

Patch
diff mbox series

diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma
index 9f645f759d2d..40efc4b3aa2e 100644
--- a/common/multipath-over-rdma
+++ b/common/multipath-over-rdma
@@ -11,6 +11,7 @@  filesystem_type=ext4
 fio_aux_path=/tmp/fio-state-files
 memtotal=$(sed -n 's/^MemTotal:[[:blank:]]*\([0-9]*\)[[:blank:]]*kB$/\1/p' /proc/meminfo)
 max_ramdisk_size=$((1<<25))
+use_siw=
 ramdisk_size=$((memtotal*(1024/16)))  # in bytes
 if [ $ramdisk_size -gt $max_ramdisk_size ]; then
 	ramdisk_size=$max_ramdisk_size
@@ -389,9 +390,13 @@  function mountpoint() {
 
 # All primary RDMA GIDs
 all_primary_gids() {
-	find /sys/devices -name infiniband | while read -r p; do
-		cat "$p"/*/ports/*/gids/0
-	done | grep -v ':0000:0000:0000:0000$'
+	local gid p
+
+	for p in /sys/class/infiniband/*/ports/*/gids/0; do
+		gid="$(<"$p")"
+		[ "$gid" != 0000:0000:0000:0000:0000:0000:0000:0000 ] &&
+			echo "$gid"
+	done
 }
 
 # Check whether or not an rdma_rxe instance has been associated with network
@@ -408,24 +413,40 @@  has_rdma_rxe() {
 	return 1
 }
 
-# Load the rdma_rxe kernel module and associate it with all network interfaces.
+# Load the rdma_rxe or siw kernel module and associate it with all network
+# interfaces.
 start_soft_rdma() {
 	{
+	if [ -n "$use_siw" ]; then
+		modprobe siw || return $?
+		(
+			cd /sys/class/net &&
+				for i in *; do
+					[ -e "$i" ] || continue
+					[ -e "/sys/class/infiniband/${i}_siw" ] && continue
+					rdma link add "${i}_siw" type siw netdev "$i" ||
+						echo "Failed to bind the siw driver to $i"
+				done
+		)
+	else
 		modprobe rdma_rxe || return $?
 		(
 			cd /sys/class/net &&
 				for i in *; do
 					if [ -e "$i" ] && ! has_rdma_rxe "$i"; then
-						echo "$i" > /sys/module/rdma_rxe/parameters/add
+						echo "$i" > /sys/module/rdma_rxe/parameters/add ||
+							echo "Failed to bind the rdma_rxe driver to $i"
 					fi
 				done
 		)
+	fi
 	} >>"$FULL"
 }
 
-# Dissociate the rdma_rxe kernel module from all network interfaces and unload
-# the rdma_rxe kernel module.
+# Dissociate the rdma_rxe or siw kernel module from all network interfaces and
+# unload the rdma_rxe kernel module.
 stop_soft_rdma() {
+	{
 	(
 		cd /sys/class/net &&
 			for i in *; do
@@ -439,6 +460,19 @@  stop_soft_rdma() {
 		echo "Unloading rdma_rxe failed"
 		return 1
 	fi
+	(
+		cd /sys/class/net &&
+			for i in *_siw; do
+				[ -e "$i" ] || continue
+				rdma link del "${i}" ||
+					echo "Failed to unbind the siw driver from ${i%_siw}"
+			done
+	)
+	if ! unload_module siw 10; then
+		echo "Unloading siw failed"
+		return 1
+	fi
+	} >>"$FULL"
 }
 
 # /dev/sd... device node assigned to the scsi_debug kernel module.
diff --git a/tests/srp/rc b/tests/srp/rc
index a1bc09b496ec..2738c8f6a4e5 100755
--- a/tests/srp/rc
+++ b/tests/srp/rc
@@ -138,30 +138,51 @@  do_ib_cm_login() {
 	done
 }
 
+rdma_dev_to_net_dev() {
+	local b d rdma_dev=$1
+
+	b=/sys/class/infiniband/$rdma_dev/parent
+	if [ -e "$b" ]; then
+		echo "$(<"$b")"
+	else
+		echo "${rdma_dev%_siw}"
+	fi
+}
+
 # Tell the SRP initiator to log in to an SRP target using the RDMA/CM.
 # Arguments: $1: SRP target IOC GUID; $2: IB device to log in to; $3: additional
 # login parameters.
 do_rdma_cm_login() {
-	local a b add_param d dest dests ibdev ioc_guid pd
+	local a b c add_param d dest dests ibdev ioc_guid params
 
 	ioc_guid=$1
 	ibdev=$2
 	add_param=$3
-	pd=/sys/class/infiniband/$ibdev/parent
-	if [ -e "$pd" ]; then
-		d=$(<"$pd")
+	if d=$(rdma_dev_to_net_dev "$ibdev"); then
 		a=$(get_ipv4_addr "$(basename "$d")")
 		b=$(get_ipv6_addr "$(basename "$d")")
 	fi
 	echo "Interface $d: IPv4 $a IPv6 $b" >>"$FULL"
 	[ -n "$a$b" ] || return 1
+	b=${b}%$(<"/sys/class/net/$d/ifindex")
 	dests=()
-	[ -n "$a" ] && dests+=("${a}:${srp_rdma_cm_port}")
-	[ -n "$b" ] && dests+=("[${b}]:${srp_rdma_cm_port}")
+	for c in $a; do
+		dests+=("${c}:${srp_rdma_cm_port}")
+	done
+	for c in $b; do
+		dests+=("[${c}]:${srp_rdma_cm_port}")
+	done
 	for dest in "${dests[@]}"; do
+		src=${dest%:*}
 		for p in "/sys/class/infiniband_srp/srp-${2}-"*; do
 			[ -e "$p" ] || continue
-			srp_single_login "id_ext=$ioc_guid,ioc_guid=$ioc_guid,dest=$dest,$add_param" "$p/add_target"
+			ibdev=$(<"$p/ibdev")
+			port=$(<"$p/port")
+			gid=$(<"/sys/class/infiniband/$ibdev/ports/$port/gids/0")
+			gid=${gid//:}
+			gid_pfx=${gid:0:16}
+			params+="id_ext=$ioc_guid,initiator_ext=$gid_pfx,ioc_guid=$ioc_guid,src=$src,dest=$dest,$add_param"
+			srp_single_login "${params}" "$p/add_target"
 		done
 	done
 }
@@ -201,11 +222,11 @@  log_in() {
 	ioc_guid=$(</sys/module/ib_srpt/parameters/srpt_service_guid)
 
 	for ((i=0;i<10;i++)); do
-		for d in /sys/class/infiniband_mad/umad*; do
-			[ -e "$d" ] || continue
-			ibdev=$(<"$d/ibdev")
-			port=$(<"$d/port")
-			link_layer=$(<"/sys/class/infiniband/$ibdev/ports/$port/link_layer")
+		for p in /sys/class/infiniband/*/ports/*; do
+			[ -e "$p" ] || continue
+			port=$(basename "$p")
+			ibdev=$(basename "$(dirname "$(dirname "$p")")")
+			link_layer=$(<"$p/link_layer")
 			case $link_layer in
 				InfiniBand)
 					do_ib_cm_login   "$ioc_guid" "$ibdev" "$port" "$add_param" ||
@@ -454,34 +475,34 @@  configure_target_ports() {
 
 # Load LIO and configure the SRP target driver and LUNs.
 start_lio_srpt() {
-	local b d gid guid i ini_gids ini_guids opts p target_gids target_guids vdev
+	local b d gid i ini_ids=() opts p target_ids=() vdev
 
-	# shellcheck disable=SC2207
-	target_guids=($(all_primary_gids | sed 's/^fe80:0000:0000:0000://'))
-	# shellcheck disable=SC2207
-	target_gids=($(all_primary_gids | sed 's/^/0x/;s/://g'))
+	for gid in $(all_primary_gids); do
+		if [ "${gid#fe8}" != "$gid" ]; then
+			gid=${gid#fe80:0000:0000:0000:}
+		else
+			gid=0x${gid//:}
+		fi
+		target_ids+=("$gid")
+	done
+	echo "target_ids=${target_ids[*]}" >>"$FULL"
 	for p in /sys/class/infiniband/*/ports/*; do
 		[ -e "$p" ] || continue
-		link_layer=$(<"$p/link_layer")
-		case "$link_layer" in
-			InfiniBand)
-				guid=$(<"$p/gids/0")
-				gid=$(echo "${guid}" | sed 's/^fe8/0x000/;s/://g')
-				guid=${guid#fe80:0000:0000:0000:}
-				[ "$guid" = "0000:0000:0000:0000" ] && continue
-				ini_guids+=("$guid")
-				ini_gids+=("$gid")
-				;;
-			*)
-				d=$(<"$(dirname "$(dirname "$p")")/parent")
-				for b in $(get_ipv4_addr "$d") \
-						 $(get_ipv6_addr "$d"|expand_ipv6_addr); do
-					ini_guids+=("$b")
-					ini_gids+=("$b")
-				done
-				;;
-		esac
+		gid=$(<"$p/gids/0")
+		if [ "${gid#fe8}" != "$gid" ]; then
+			gid=${gid#fe80:0000:0000:0000:}
+			ini_ids+=("$gid")
+		else
+			gid="0x${gid//:}"
+			ini_ids+=("$gid")
+		fi
+		d=$(rdma_dev_to_net_dev "$(basename "$(dirname "$(dirname "$p")")")")
+		for b in $(get_ipv4_addr "$d") \
+				 $(get_ipv6_addr "$d"|expand_ipv6_addr); do
+			ini_ids+=("$b")
+		done
 	done
+	echo "ini_ids=${ini_ids[*]}" >>"$FULL"
 	mount_configfs || return $?
 	modprobe target_core_mod || return $?
 	modprobe target_core_iblock || return $?
@@ -519,10 +540,7 @@  start_lio_srpt() {
 			echo "${srp_rdma_cm_port}" > discovery_auth/rdma_cm_port ||
 				return $?
 		fi
-		configure_target_ports "${target_guids[@]}" -- "${ini_guids[@]}" || {
-			echo "Retrying with old port name format"
-			configure_target_ports "${target_gids[@]}" -- "${ini_gids[@]}"
-		}
+		configure_target_ports "${target_ids[@]}" -- "${ini_ids[@]}"
 	)
 }