| Message ID | 20210121035954.7245-1-bvanassche@acm.org (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | [blktests] rdma: Use rdma link instead of /sys/class/infiniband/*/parent | expand |
On 1/21/21 11:59 AM, Bart Van Assche wrote: > The approach of verifying whether or not an RDMA interface is associated > with the rdma_rxe interface by looking up its parent device is deprecated > and will be removed soon. Hence this patch that uses the rdma link command > instead. > > Cc: Jason Gunthorpe <jgg@nvidia.com> > Cc: Yi Zhang <yi.zhang@redhat.com> > Signed-off-by: Bart Van Assche <bvanassche@acm.org> > --- > common/multipath-over-rdma | 107 ++++++++++--------------------------- > tests/srp/rc | 9 +--- > 2 files changed, 28 insertions(+), 88 deletions(-) > > diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma > index 9d9d2b27af83..b85e31e7bce1 100644 > --- a/common/multipath-over-rdma > +++ b/common/multipath-over-rdma > @@ -76,41 +76,9 @@ is_number() { > [ "$1" -eq "0$1" ] 2>/dev/null > } > > -# Check whether a device is an RDMA device. An example argument: > -# /sys/devices/pci0000:00/0000:00:03.0/0000:04:00.0 > -is_rdma_device() { > - local d i inode1 inode2 > - > - inode1=$(stat -c %i "$1") > - # echo "inode1 = $inode1" > - for i in /sys/class/infiniband/*; do > - d=/sys/class/infiniband/"$(readlink "$i")" > - d=$(dirname "$(dirname "$d")") > - inode2=$(stat -c %i "$d") > - # echo "inode2 = $inode2" > - if [ "$inode1" = "$inode2" ]; then > - return > - fi > - done > - false > -} > - > # Lists RDMA capable network interface names, e.g. ib0 ib1. > rdma_network_interfaces() { > - ( > - cd /sys/class/net && > - for i in *; do > - [ -e "$i" ] || continue > - # Skip IPoIB (ARPHRD_INFINIBAND) network > - # interfaces. > - [ "$(<"$i"/type)" = 32 ] && continue > - [ -L "$i/device" ] || continue > - d=$(readlink "$i/device" 2>/dev/null) > - if [ -n "$d" ] && is_rdma_device "$i/$d"; then > - echo "$i" > - fi > - done > - ) > + rdma link show | sed -n 's,^link[[:blank:]]*\([^/]*\)/.*,\1,p' | sort -u We should list the network interface here(like eno1), rxe/siw interfaces will not work for function get_ipv4_addr # rdma link link eno1_rxe/1 state ACTIVE physical_state LINK_UP netdev eno1 link eno1_siw/1 state ACTIVE physical_state LINK_UP netdev eno1 # rdma link show | sed -n 's,^link[[:blank:]]*\([^/]*\)/.*,\1,p' | sort -u eno1_rxe eno1_siw for i in $(rdma_network_interfaces) do ipv4_addr=$(get_ipv4_addr "$i") if [ -n "${ipv4_addr}" ]; then def_traddr=${ipv4_addr} fi done > } > > # Check whether any stacked block device holds block device $1. If so, echo > @@ -411,47 +379,36 @@ all_primary_gids() { > done > } > > -# Check whether or not an rdma_rxe instance has been associated with network > -# interface $1. > -has_rdma_rxe() { > - local f > - > - for f in /sys/class/infiniband/*/parent; do > - if [ -e "$f" ] && [ "$(<"$f")" = "$1" ]; then > - return 0 > - fi > - done > - > - return 1 > +# Check whether or not an rdma_rxe or siw instance has been associated with > +# network interface $1. > +has_soft_rdma() { > + rdma link | grep -q " netdev $1[[:blank:]]*\$" > } > > # Load the rdma_rxe or siw kernel module and associate it with all network > # interfaces. > start_soft_rdma() { > + local type > + > { > if [ -n "$use_siw" ]; then > modprobe siw || return $? > - ( > - cd /sys/class/net && > - for i in *; do > - [ -e "$i" ] || continue > - [ -e "/sys/class/infiniband/${i}_siw" ] && continue > - rdma link add "${i}_siw" type siw netdev "$i" || > - echo "Failed to bind the siw driver to $i" > - done > - ) > + type=siw > else > modprobe rdma_rxe || return $? > - ( > - cd /sys/class/net && > - for i in *; do > - if [ -e "$i" ] && ! has_rdma_rxe "$i"; then > - echo "$i" > /sys/module/rdma_rxe/parameters/add || > - echo "Failed to bind the rdma_rxe driver to $i" > - fi > - done > - ) > + type=rxe > fi > + ( > + cd /sys/class/net && > + for i in *; do > + [ -e "$i" ] || continue > + [ "$i" = "lo" ] && continue > + [ "$(<"$i/addr_len")" = 6 ] || continue > + has_soft_rdma "$i" && continue > + rdma link add "${i}_$type" type $type netdev "$i" || > + echo "Failed to bind the $type driver to $i" > + done > + ) > } >>"$FULL" > } > > @@ -459,27 +416,17 @@ start_soft_rdma() { > # unload the rdma_rxe kernel module. > stop_soft_rdma() { > { > - ( > - cd /sys/class/net && > - for i in *; do > - if [ -e "$i" ] && has_rdma_rxe "$i"; then > - { echo "$i" > /sys/module/rdma_rxe/parameters/remove; } \ > - 2>/dev/null > - fi > - done > - ) > + rdma link | > + sed -n 's,^link[[:blank:]]*\([^/]*\)/.* netdev .*,\1,p' | > + while read -r i; do > + echo "$i ..." > + rdma link del "${i}" || > + echo "Failed to unbind siw/rxe from ${i}" > + done > if ! unload_module rdma_rxe 10; then > echo "Unloading rdma_rxe failed" > return 1 > fi > - ( > - cd /sys/class/net && > - for i in *_siw; do > - [ -e "$i" ] || continue > - rdma link del "${i}" || > - echo "Failed to unbind the siw driver from ${i%_siw}" > - done > - ) > if ! unload_module siw 10; then > echo "Unloading siw failed" > return 1 > diff --git a/tests/srp/rc b/tests/srp/rc > index 1f665a28db66..b8ac9e27a2fd 100755 > --- a/tests/srp/rc > +++ b/tests/srp/rc > @@ -142,14 +142,7 @@ do_ib_cm_login() { > } > > rdma_dev_to_net_dev() { > - local b d rdma_dev=$1 > - > - b=/sys/class/infiniband/$rdma_dev/parent > - if [ -e "$b" ]; then > - echo "$(<"$b")" > - else > - echo "${rdma_dev%_siw}" > - fi > + rdma link show "$1/1" | sed 's/.* netdev //;s/[[:blank:]]*$//' > } > > # Tell the SRP initiator to log in to an SRP target using the RDMA/CM. >
On 1/20/21 10:58 PM, Yi Zhang wrote: > On 1/21/21 11:59 AM, Bart Van Assche wrote: >> rdma_network_interfaces() { >> - ( >> - cd /sys/class/net && >> - for i in *; do >> - [ -e "$i" ] || continue >> - # Skip IPoIB (ARPHRD_INFINIBAND) network >> - # interfaces. >> - [ "$(<"$i"/type)" = 32 ] && continue >> - [ -L "$i/device" ] || continue >> - d=$(readlink "$i/device" 2>/dev/null) >> - if [ -n "$d" ] && is_rdma_device "$i/$d"; then >> - echo "$i" >> - fi >> - done >> - ) >> + rdma link show | sed -n 's,^link[[:blank:]]*\([^/]*\)/.*,\1,p' | >> sort -u > > We should list the network interface here(like eno1), rxe/siw interfaces > will not work for function get_ipv4_addr Thanks for the feedback. I will fix this and repost this patch. Bart.
diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma index 9d9d2b27af83..b85e31e7bce1 100644 --- a/common/multipath-over-rdma +++ b/common/multipath-over-rdma @@ -76,41 +76,9 @@ is_number() { [ "$1" -eq "0$1" ] 2>/dev/null } -# Check whether a device is an RDMA device. An example argument: -# /sys/devices/pci0000:00/0000:00:03.0/0000:04:00.0 -is_rdma_device() { - local d i inode1 inode2 - - inode1=$(stat -c %i "$1") - # echo "inode1 = $inode1" - for i in /sys/class/infiniband/*; do - d=/sys/class/infiniband/"$(readlink "$i")" - d=$(dirname "$(dirname "$d")") - inode2=$(stat -c %i "$d") - # echo "inode2 = $inode2" - if [ "$inode1" = "$inode2" ]; then - return - fi - done - false -} - # Lists RDMA capable network interface names, e.g. ib0 ib1. rdma_network_interfaces() { - ( - cd /sys/class/net && - for i in *; do - [ -e "$i" ] || continue - # Skip IPoIB (ARPHRD_INFINIBAND) network - # interfaces. - [ "$(<"$i"/type)" = 32 ] && continue - [ -L "$i/device" ] || continue - d=$(readlink "$i/device" 2>/dev/null) - if [ -n "$d" ] && is_rdma_device "$i/$d"; then - echo "$i" - fi - done - ) + rdma link show | sed -n 's,^link[[:blank:]]*\([^/]*\)/.*,\1,p' | sort -u } # Check whether any stacked block device holds block device $1. If so, echo @@ -411,47 +379,36 @@ all_primary_gids() { done } -# Check whether or not an rdma_rxe instance has been associated with network -# interface $1. -has_rdma_rxe() { - local f - - for f in /sys/class/infiniband/*/parent; do - if [ -e "$f" ] && [ "$(<"$f")" = "$1" ]; then - return 0 - fi - done - - return 1 +# Check whether or not an rdma_rxe or siw instance has been associated with +# network interface $1. +has_soft_rdma() { + rdma link | grep -q " netdev $1[[:blank:]]*\$" } # Load the rdma_rxe or siw kernel module and associate it with all network # interfaces. start_soft_rdma() { + local type + { if [ -n "$use_siw" ]; then modprobe siw || return $? - ( - cd /sys/class/net && - for i in *; do - [ -e "$i" ] || continue - [ -e "/sys/class/infiniband/${i}_siw" ] && continue - rdma link add "${i}_siw" type siw netdev "$i" || - echo "Failed to bind the siw driver to $i" - done - ) + type=siw else modprobe rdma_rxe || return $? - ( - cd /sys/class/net && - for i in *; do - if [ -e "$i" ] && ! has_rdma_rxe "$i"; then - echo "$i" > /sys/module/rdma_rxe/parameters/add || - echo "Failed to bind the rdma_rxe driver to $i" - fi - done - ) + type=rxe fi + ( + cd /sys/class/net && + for i in *; do + [ -e "$i" ] || continue + [ "$i" = "lo" ] && continue + [ "$(<"$i/addr_len")" = 6 ] || continue + has_soft_rdma "$i" && continue + rdma link add "${i}_$type" type $type netdev "$i" || + echo "Failed to bind the $type driver to $i" + done + ) } >>"$FULL" } @@ -459,27 +416,17 @@ start_soft_rdma() { # unload the rdma_rxe kernel module. stop_soft_rdma() { { - ( - cd /sys/class/net && - for i in *; do - if [ -e "$i" ] && has_rdma_rxe "$i"; then - { echo "$i" > /sys/module/rdma_rxe/parameters/remove; } \ - 2>/dev/null - fi - done - ) + rdma link | + sed -n 's,^link[[:blank:]]*\([^/]*\)/.* netdev .*,\1,p' | + while read -r i; do + echo "$i ..." + rdma link del "${i}" || + echo "Failed to unbind siw/rxe from ${i}" + done if ! unload_module rdma_rxe 10; then echo "Unloading rdma_rxe failed" return 1 fi - ( - cd /sys/class/net && - for i in *_siw; do - [ -e "$i" ] || continue - rdma link del "${i}" || - echo "Failed to unbind the siw driver from ${i%_siw}" - done - ) if ! unload_module siw 10; then echo "Unloading siw failed" return 1 diff --git a/tests/srp/rc b/tests/srp/rc index 1f665a28db66..b8ac9e27a2fd 100755 --- a/tests/srp/rc +++ b/tests/srp/rc @@ -142,14 +142,7 @@ do_ib_cm_login() { } rdma_dev_to_net_dev() { - local b d rdma_dev=$1 - - b=/sys/class/infiniband/$rdma_dev/parent - if [ -e "$b" ]; then - echo "$(<"$b")" - else - echo "${rdma_dev%_siw}" - fi + rdma link show "$1/1" | sed 's/.* netdev //;s/[[:blank:]]*$//' } # Tell the SRP initiator to log in to an SRP target using the RDMA/CM.
The approach of verifying whether or not an RDMA interface is associated with the rdma_rxe interface by looking up its parent device is deprecated and will be removed soon. Hence this patch that uses the rdma link command instead. Cc: Jason Gunthorpe <jgg@nvidia.com> Cc: Yi Zhang <yi.zhang@redhat.com> Signed-off-by: Bart Van Assche <bvanassche@acm.org> --- common/multipath-over-rdma | 107 ++++++++++--------------------------- tests/srp/rc | 9 +--- 2 files changed, 28 insertions(+), 88 deletions(-)