new file mode 100644
@@ -0,0 +1 @@
+multipath.conf
new file mode 100755
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Log in and log out"
+QUICK=1
+
+count_devices() {
+ local d devs=0
+
+ for d in /sys/class/nvme-fabrics/ctl/*/*/device; do
+ [ -d "$d" ] && ((devs++))
+ done
+ echo $devs
+}
+
+wait_for_devices() {
+ local expected=1 i devices
+
+ use_blk_mq y || return $?
+ for ((i=0;i<100;i++)); do
+ devices=$(count_devices)
+ [ "$devices" -ge $expected ] && break
+ sleep .1
+ done
+ echo "count_devices(): $devices <> $expected" >>"$FULL"
+ echo "count_devices(): $devices <> $expected"
+ [ "$devices" -ge $expected ]
+}
+
+test() {
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && wait_for_devices && echo Passed
+}
new file mode 100644
@@ -0,0 +1,3 @@
+Configured NVMe target driver
+count_devices(): 1 <> 1
+Passed
new file mode 100755
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="File I/O on top of multipath concurrently with logout and login (mq)"
+TIMED=1
+
+test_disconnect_repeatedly() {
+ local dev fio_status m
+
+ use_blk_mq y || return $?
+ dev=$(get_bdev 0) || return $?
+ m=$(mountpoint 0) || return $?
+ create_filesystem "$dev" || return $?
+ mount_and_check "$dev" "$m" || return $?
+ # shellcheck disable=SC2064
+ trap "unmount_and_check $m" RETURN
+ simulate_network_failure_loop "$dev" "$TIMEOUT" &
+ run_fio --verify=md5 --rw=randwrite --bs=4K --loops=$((10**6)) \
+ --iodepth=64 --group_reporting --sync=1 --direct=1 \
+ --ioengine=libaio --directory="$m" --runtime="${TIMEOUT}" \
+ --name=data-integrity-test-mq --thread --numjobs=16 \
+ --output="${RESULTS_DIR}/nvmeof-mp/fio-output-002.txt" \
+ >>"$FULL"
+ fio_status=$?
+ wait
+ return $fio_status
+}
+
+test() {
+ : "${TIMEOUT:=30}"
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_disconnect_repeatedly && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="File I/O on top of multipath concurrently with logout and login (sq-on-mq)"
+TIMED=1
+
+test_disconnect_repeatedly() {
+ local dev fio_status m
+
+ use_blk_mq n || return $?
+ dev=$(get_bdev 0) || return $?
+ m=$(mountpoint 0) || return $?
+ create_filesystem "$dev" || return $?
+ mount_and_check "$dev" "$m" || return $?
+ # shellcheck disable=SC2064
+ trap "unmount_and_check $m" RETURN
+ simulate_network_failure_loop "$dev" "$TIMEOUT" &
+ run_fio --verify=md5 --rw=randwrite --bs=4K --loops=$((10**6)) \
+ --iodepth=64 --group_reporting --sync=1 --direct=1 \
+ --ioengine=libaio --directory="$m" \
+ --name=data-integrity-test-02-sq-on-mq --thread \
+ --numjobs=16 --runtime="${TIMEOUT}" \
+ --output="${RESULTS_DIR}/nvmeof-mp/fio-output-004.txt" >>"$FULL"
+ fio_status=$?
+ wait
+ return $fio_status
+}
+
+test() {
+ : "${TIMEOUT:=30}"
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_disconnect_repeatedly && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Direct I/O with large transfer sizes and bs=4M"
+QUICK=1
+
+test_large_transfer_size() {
+ local dev m
+
+ use_blk_mq y || return $?
+ dev=$(get_bdev 0) || return $?
+ run_fio --verify=md5 --rw=randwrite --bs=4M --loops=$((10**6)) \
+ --iodepth=4 --group_reporting --sync=1 --direct=1 \
+ --ioengine=libaio \
+ --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+ --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-005.txt" \
+ >>"$FULL"
+}
+
+test() {
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_large_transfer_size && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Direct I/O with large transfer sizes and bs=8M"
+QUICK=1
+
+test_large_transfer_size() {
+ local dev m
+
+ use_blk_mq y || return $?
+ dev=$(get_bdev 0) || return $?
+ run_fio --verify=md5 --rw=randwrite --bs=8M --loops=$((10**6)) \
+ --iodepth=4 --group_reporting --sync=1 --direct=1 \
+ --ioengine=libaio \
+ --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+ --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-006.txt" \
+ >>"$FULL"
+}
+
+test() {
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_large_transfer_size && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Buffered I/O with large transfer sizes and bs=4M"
+QUICK=1
+
+test_large_transfer_size() {
+ local dev m
+
+ use_blk_mq y || return $?
+ dev=$(get_bdev 0) || return $?
+ run_fio --verify=md5 --rw=randwrite --bs=4M --loops=$((10**6)) \
+ --iodepth=4 --group_reporting --sync=1 --direct=0 \
+ --ioengine=libaio \
+ --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+ --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-009.txt" \
+ >>"$FULL"
+}
+
+test() {
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_large_transfer_size && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Buffered I/O with large transfer sizes and bs=8M"
+QUICK=1
+
+test_large_transfer_size() {
+ local dev m
+
+ use_blk_mq y || return $?
+ dev=$(get_bdev 0) || return $?
+ run_fio --verify=md5 --rw=randwrite --bs=8M --loops=$((10**6)) \
+ --iodepth=4 --group_reporting --sync=1 --direct=0 \
+ --ioengine=libaio \
+ --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+ --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-010.txt" \
+ >>"$FULL"
+}
+
+test() {
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_large_transfer_size && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Block I/O on top of multipath concurrently with logout and login"
+TIMED=1
+
+test_disconnect_repeatedly() {
+ local dev fio_status m
+
+ use_blk_mq y || return $?
+ dev=$(get_bdev 0) || return $?
+ simulate_network_failure_loop "$dev" "$TIMEOUT" &
+ run_fio --verify=md5 --rw=randwrite --bs=4K --loops=10000 \
+ --ioengine=libaio --iodepth=64 --iodepth_batch=32 \
+ --group_reporting --sync=1 --direct=1 --filename="$dev" \
+ --name=data-integrity-test-06 --thread --numjobs=1 \
+ --runtime="${TIMEOUT}" \
+ --output="${RESULTS_DIR}/nvmeof-mp/fio-output-011.txt" \
+ >>"$FULL"
+ fio_status=$?
+ wait
+ return $fio_status
+}
+
+test() {
+ : "${TIMEOUT:=30}"
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_disconnect_repeatedly && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="dm-mpath on top of multiple I/O schedulers"
+QUICK=1
+
+test_io_schedulers() {
+ local dev m
+
+ # Load all I/O scheduler kernel modules
+ for m in "/lib/modules/$(uname -r)/kernel/block/"*.ko; do
+ modprobe "$(basename "$m")" >&/dev/null
+ done
+ for mq in y n; do
+ use_blk_mq ${mq} || return $?
+ dev=$(get_bdev 0) || return $?
+ for sched in noop deadline bfq cfq; do
+ set_scheduler "$(basename "$(readlink -f "${dev}")")" $sched \
+ >>"$FULL" 2>&1 || continue
+ echo "I/O scheduler: $sched; use_blk_mq: $mq" >>"$FULL"
+ run_fio --verify=md5 --rw=randwrite --bs=4K --size=64K \
+ --ioengine=libaio --iodepth=64 \
+ --iodepth_batch=32 --group_reporting --sync=1 \
+ --direct=1 --filename="$dev" \
+ --name=${sched} --thread --numjobs=1 \
+ --output="${RESULTS_DIR}/nvmeof-mp/012-${sched}-${mq}.txt" \
+ >>"$FULL" ||
+ return $?
+ done
+ done
+}
+
+test() {
+ trap 'trap "" EXIT; teardown' EXIT
+ setup && test_io_schedulers && echo Passed
+}
new file mode 100644
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
new file mode 100755
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ "${1#nvme}" != "$1" ]; then
+ wwid=$(<"/sys/block/$1/wwid")
+ echo "${wwid#uuid.}"
+else
+ for e in /usr/lib/udev/scsi_id /lib/udev/scsi_id; do
+ [ -e "$e" ] && break
+ done
+ "$e" --whitelisted "/dev/$1"
+fi
new file mode 100644
@@ -0,0 +1,28 @@
+defaults {
+ find_multipaths no
+ user_friendly_names yes
+ queue_without_daemon no
+ getuid_callout "$PWD/tests/nvmeof-mp/getuid_callout %n"
+}
+devices {
+ device {
+ vendor "LIO-ORG|SCST_BIO|FUSIONIO"
+ product ".*"
+ features "1 queue_if_no_path"
+ path_checker tur
+ }
+}
+blacklist {
+ device {
+ vendor "ATA|QEMU"
+ }
+ device {
+ vendor "Linux"
+ product "scsi_debug"
+ }
+ devnode "^nullb.*"
+}
+blacklist_exceptions {
+ property ".*"
+ devnode "^nvme"
+}
new file mode 100755
@@ -0,0 +1,936 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. common/rc
+
+namespace=(1)
+memtotal=$(sed -n 's/^MemTotal:[[:blank:]]*\([0-9]*\)[[:blank:]]*kB$/\1/p' /proc/meminfo)
+max_ramdisk_size=$((1<<25))
+ramdisk_size=$((memtotal*(1024/16))) # in bytes
+if [ $ramdisk_size -gt $max_ramdisk_size ]; then
+ ramdisk_size=$max_ramdisk_size
+fi
+elevator=none
+filesystem_type=ext4
+fio_aux_path=/tmp/fio-state-files
+nvme_subsysnqn="nvme-test"
+nvme_port=7777
+ini_timeout=1
+
+expand() {
+ sed "s,\$PWD,$PWD,g" <"$1" >"$2"
+}
+
+group_requires() {
+ local m name p required_modules
+
+ # Since the nvmeof-mp tests are based on the dm-mpath driver, these
+ # tests are incompatible with the NVME_MULTIPATH kernel configuration
+ # option.
+ _have_kernel_option NVME_MULTIPATH && exit 1
+
+ _have_configfs || return $?
+ required_modules=(
+ dm_multipath
+ dm_queue_length
+ dm_service_time
+ null_blk
+ rdma_cm
+ ib_ipoib
+ ib_umad
+ nvme-rdma
+ nvmet-rdma
+ rdma_rxe
+ scsi_dh_alua
+ scsi_dh_emc
+ scsi_dh_rdac
+ )
+ for m in "${required_modules[@]}"; do
+ _have_module "$m" || return $?
+ done
+
+ for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof sg_reset; do
+ _have_program "$p" || return $?
+ done
+
+ _have_root || return $?
+
+ _have_kernel_option DM_UEVENT || return $?
+
+ for name in multipathd multipathd; do
+ if pidof "$name" >/dev/null; then
+ SKIP_REASON="$name must be stopped before the nvmeof-mp tests are run"
+ return 1
+ fi
+ done
+ if [ tests/nvmeof-mp/multipath.conf.in -nt \
+ tests/nvmeof-mp/multipath.conf ]; then
+ expand tests/nvmeof-mp/multipath.conf.in \
+ tests/nvmeof-mp/multipath.conf
+ fi
+ if [ -e /etc/multipath.conf ] &&
+ ! diff -q /etc/multipath.conf tests/nvmeof-mp/multipath.conf >&/dev/null
+ then
+ SKIP_REASON="/etc/multipath.conf already exists"
+ return 1
+ fi
+}
+
+# Log out, set dm and SCSI use_blk_mq parameters and log in. $1: device mapper
+# use_blk_mq mode; $2..${$#}: NVMeOF initiator kernel module parameters.
+use_blk_mq() {
+ local dm_mode=$1 kmod_params
+
+ shift
+ kmod_params=("$@")
+
+ (
+ cd /sys/module/dm_mod/parameters || return $?
+ if [ -e use_blk_mq ]; then
+ echo "$dm_mode" >use_blk_mq || return $?
+ fi
+ )
+
+ log_out &&
+ remove_mpath_devs &&
+ stop_client &&
+ start_client "${kmod_params[@]}" &&
+ log_in
+}
+
+get_ipv4_addr() {
+ ip -4 -o addr show dev "$1" |
+ sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p'
+}
+
+# Convert e.g. ::1 into 0000:0000:0000:0000:0000:0000:0000:0001.
+expand_ipv6_addr() {
+ awk -F : 'BEGIN{left=1} { for(i=1;i<=NF;i++) { a=substr("0000", 1+length($i)) $i; if ($i == "") left=0; else if (left) pre = pre ":" a; else suf = suf ":" a }; mid=substr(":0000:0000:0000:0000:0000:0000:0000:0000", (pre!="")+length(pre)+length(suf)); print substr(pre,2) mid suf}'
+}
+
+get_ipv6_addr() {
+ ip -6 -o addr show dev "$1" |
+ sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p'
+}
+
+# Whether or not $1 is a number.
+is_number() {
+ [ "$1" -eq "0$1" ] 2>/dev/null
+}
+
+# Check whether a device is an RDMA device. An example argument:
+# /sys/devices/pci0000:00/0000:00:03.0/0000:04:00.0
+is_rdma_device() {
+ local d i inode1 inode2
+
+ inode1=$(stat -c %i "$1")
+ # echo "inode1 = $inode1"
+ for i in /sys/class/infiniband/*; do
+ d=/sys/class/infiniband/"$(readlink "$i")"
+ d=$(dirname "$(dirname "$d")")
+ inode2=$(stat -c %i "$d")
+ # echo "inode2 = $inode2"
+ if [ "$inode1" = "$inode2" ]; then
+ return
+ fi
+ done
+ false
+}
+
+# Lists RDMA capable network interface names, e.g. ib0 ib1.
+rdma_network_interfaces() {
+ (
+ cd /sys/class/net &&
+ for i in *; do
+ [ -e "$i" ] || continue
+ # Skip IPoIB (ARPHRD_INFINIBAND) network
+ # interfaces.
+ [ "$(<"$i"/type)" = 32 ] && continue
+ [ -L "$i/device" ] || continue
+ d=$(readlink "$i/device" 2>/dev/null)
+ if [ -n "$d" ] && is_rdma_device "$i/$d"; then
+ echo "$i"
+ fi
+ done
+ )
+}
+
+log_in() {
+ local i ipv4_addr
+
+ [ -c /dev/nvme-fabrics ] &&
+ for i in $(rdma_network_interfaces); do
+ ipv4_addr=$(get_ipv4_addr "$i")
+ if [ -n "${ipv4_addr}" ]; then
+ { echo -n "transport=rdma,traddr=${ipv4_addr},trsvcid=${nvme_port},nqn=$nvme_subsysnqn" > /dev/nvme-fabrics; } 2>>"$FULL"
+ fi
+ done &&
+ echo reconfigure | multipathd -k >&/dev/null
+}
+
+log_out() {
+ local c
+
+ for c in /sys/class/nvme-fabrics/ctl/*/delete_controller; do
+ [ -e "$c" ] && echo 1 > "$c" &
+ done
+ wait
+}
+
+# Check whether any stacked block device holds block device $1. If so, echo
+# the name of the holder.
+held_by() {
+ local d e dev=$1
+
+ while [ -L "$dev" ]; do
+ dev=$(realpath "$dev")
+ done
+ dev=${dev%/dev/}
+ for d in /sys/class/block/*/holders/*; do
+ [ -e "$d" ] || continue
+ e=$(basename "$d")
+ if [ "$e" = "$dev" ]; then
+ echo "/dev/$(basename "$(dirname "$(dirname "$d")")")"
+ fi
+ done
+}
+
+# System uptime in seconds.
+uptime_s() {
+ local a b
+
+ echo "$(</proc/uptime)" | { read -r a b && echo "${a%%.*}"; }
+}
+
+# Sleep until either $1 seconds have elapsed or until the deadline $2 has been
+# reached. Return 1 if and only if the deadline has been met.
+sleep_until() {
+ local duration=$1 deadline=$2 u
+
+ u=$(uptime_s)
+ if [ $((u + duration)) -le "$deadline" ]; then
+ sleep "$duration"
+ else
+ [ "$deadline" -gt "$u" ] && sleep $((deadline - u))
+ return 1
+ fi
+}
+
+# Simulate network failures for device $1 during $2 seconds.
+simulate_network_failure_loop() {
+ local d dev="$1" duration="$2" deadline i rc=0 s
+
+ [ -e "$dev" ] || return $?
+ [ -n "$duration" ] || return $?
+ deadline=$(($(uptime_s) + duration))
+ s=5
+ while [ $rc = 0 ]; do
+ sleep_until 5 ${deadline} || break
+ for d in $(held_by "$dev"); do
+ echo 1 >"$d/device/reset_controller"
+ done
+ done
+
+ for ((i=0;i<5;i++)); do
+ log_in && break
+ sleep 1
+ done
+}
+
+# Kill all processes that have opened block device $1.
+stop_bdev_users() {
+ [ -n "$1" ] || return $?
+ lsof -F p "$1" 2>/dev/null | while read -r line; do
+ p="${line#p}"
+ if [ "$p" != "$line" ]; then
+ echo -n " (pid $p)" >>"$FULL"
+ kill -9 "$p"
+ fi
+ done
+}
+
+# RHEL 6 dmsetup accepts mpath<n> but not /dev/dm-<n> as its first argument.
+# Hence this function that converts /dev/dm-<n> into mpath<n>.
+dev_to_mpath() {
+ local d e mm
+
+ d="${1#/dev/mapper/}";
+ if [ "$d" != "$1" ]; then
+ echo "$d"
+ return 0
+ fi
+
+ [ -e "$1" ] || return $?
+
+ if [ -L "$1" ]; then
+ e=$(readlink -f "$1")
+ else
+ e="$1"
+ fi
+ if ! mm=$(stat -c %t:%T "$e"); then
+ echo "stat $1 -> $e failed"
+ return 1
+ fi
+
+ for d in /dev/mapper/mpath*; do
+ if [ -L "$d" ]; then
+ e=$(readlink -f "$d")
+ elif [ -e "$d" ]; then
+ e="$d"
+ else
+ continue
+ fi
+ if [ "$(stat -c %t:%T "$e")" = "$mm" ]; then
+ basename "$d"
+ return 0
+ fi
+ done
+ return 1
+}
+
+# Modify mpath device $1 to fail_if_no_path mode, unmount the filesystem on top
+# of it and remove the mpath device.
+remove_mpath_dev() {
+ local cmd dm i output t1 t2
+
+ {
+ for ((i=10;i>0;i--)); do
+ cmd="dm=\$(dev_to_mpath \"$1\")"
+ if ! eval "$cmd"; then
+ echo "$cmd: failed"
+ else
+ t1=$(dmsetup table "$dm")
+ cmd="dmsetup message $dm 0 fail_if_no_path"
+ if ! eval "$cmd"; then
+ echo "$cmd: failed"
+ else
+ t2=$(dmsetup table "$dm")
+ if echo "$t2" | grep -qw queue_if_no_path; then
+ echo "$dm: $t1 -> $t2"
+ fi
+ echo "Attempting to unmount /dev/mapper/$dm"
+ umount "/dev/mapper/$dm"
+ cmd="dmsetup remove $dm"
+ if ! output=$(eval "$cmd" 2>&1); then
+ echo "$cmd: $output; retrying"
+ else
+ echo "done"
+ break
+ fi
+ fi
+ fi
+ if [ ! -e "$1" ]; then
+ break
+ fi
+ ls -l "$1"
+ stop_bdev_users "$(readlink -f "$1")"
+ sleep .5
+ done
+ if [ $i = 0 ]; then
+ echo "failed"
+ return 1
+ fi
+ } &>>"$FULL"
+}
+
+# Check whether one or more arguments contain stale device nodes (/dev/...).
+mpath_has_stale_dev() {
+ local d
+
+ for d in "$@"; do
+ if [ "${d/://}" != "$d" ]; then
+ grep -qw "$d" /sys/class/block/*/dev 2>/dev/null ||
+ return 0
+ fi
+ done
+
+ return 1
+}
+
+# Check whether multipath definition $1 includes the queue_if_no_path keyword.
+is_qinp_def() {
+ case "$1" in
+ " 3 queue_if_no_path queue_mode mq ")
+ return 0;;
+ " 1 queue_if_no_path ")
+ return 0;;
+ *)
+ return 1;;
+ esac
+}
+
+remove_nvme_mpath_devs() {
+ local dm h
+
+ for h in /sys/class/block/nvme*/holders/*; do
+ [ -e "$h" ] || continue
+ d=$(basename "$(dirname "$(dirname "$h")")")
+ dm=/dev/$(basename "$h")
+ {
+ echo -n "NVME dev $d: removing $dm: "
+ dmsetup remove "$(dev_to_mpath "$dm")" && echo "done"
+ } &>> "$FULL"
+ done
+ {
+ # Find all multipaths with one or more deleted devices and
+ # remove these
+ echo "Examining all multipaths"
+ dmsetup table | while read -r mpdev fs ls type def; do
+ echo "$fs $ls" >/dev/null
+ # shellcheck disable=SC2086
+ if [ "$type" = multipath ] &&
+ { is_qinp_def "$def" ||
+ mpath_has_stale_dev $def; }; then
+ echo "${mpdev%:}"
+ fi
+ done |
+ sort -u |
+ while read -r mpdev; do
+ mpdev="/dev/mapper/$mpdev"
+ echo -n "removing $mpdev: "
+ if ! remove_mpath_dev "$mpdev"; then
+ echo "failed"
+ fi
+ done
+ echo "Finished examining multipaths"
+ } &>> "$FULL"
+}
+
+remove_mpath_devs() {
+ remove_nvme_mpath_devs
+}
+
+# Arguments: module to unload ($1) and retry count ($2).
+unload_module() {
+ local i m=$1 rc=${2:-1}
+
+ [ ! -e "/sys/module/$m" ] && return 0
+ for ((i=rc;i>0;i--)); do
+ modprobe -r "$m"
+ [ ! -e "/sys/module/$m" ] && return 0
+ sleep .1
+ done
+ return 1
+}
+
+start_nvme_client() {
+ modprobe nvme dyndbg=+pmf &&
+ modprobe nvme-core dyndbg=+pmf &&
+ modprobe nvme-fabrics dyndbg=+pmf &&
+ modprobe nvme-rdma dyndbg=+pmf &&
+ mkdir -p "$(mountpoint 0)"
+}
+
+stop_nvme_client() {
+ unload_module nvme_rdma &&
+ unload_module nvme
+}
+
+# Load the initiator kernel driver with kernel module parameters $1..$n.
+start_client() {
+ start_nvme_client "$@"
+}
+
+stop_client() {
+ stop_nvme_client
+}
+
+# Load the configfs kernel module and mount it.
+mount_configfs() {
+ if [ ! -e /sys/module/configfs ]; then
+ modprobe configfs || return $?
+ fi
+ if ! mount | grep -qw configfs; then
+ mount -t configfs none /sys/kernel/config || return $?
+ fi
+}
+
+# Get the name of the initiator device node that communicates with target
+# device $1. $1 is an index in the $namespace array.
+get_nvme_bdev() {
+ (
+ cd /sys/kernel/config/nvmet/subsystems || return $?
+ uuid=$(<"${nvme_subsysnqn}/namespaces/${namespace[$1]}/device_uuid")
+ by_id=/dev/disk/by-id/nvme-uuid.$uuid
+ [ -e "${by_id}" ] && readlink -f "${by_id}"
+ )
+}
+
+# Get a the uuid or wwid of device $1. $1 is an index in the $namespace array.
+get_bdev_uid() {
+ local i=$1
+
+ is_number "$i" || return $?
+ bdev=$(get_nvme_bdev "$i") || return $?
+ echo "$(<"/sys/block/${bdev#/dev/}/wwid")"
+}
+
+# Set scheduler of block device $1 to $2.
+set_scheduler() {
+ local b=$1 p s=$2
+
+ p=/sys/class/block/$b/queue/scheduler
+ if [ -e "/sys/block/$b/mq" ]; then
+ case "$s" in
+ noop) s=none;;
+ deadline) s=mq-deadline;;
+ bfq) s=bfq;;
+ esac
+ else
+ case "$s" in
+ none) s=noop;;
+ mq-deadline) s=deadline;;
+ bfq-mq) s=bfq;;
+ esac
+ fi
+ if ! echo "$s" > "$p"; then
+ echo "Changing scheduler of $b from $(<"$p") into $s failed"
+ return 1
+ fi
+}
+
+# Get a /dev/... path that points at dm device number $1. $1 is an index in
+# the $namespace array.
+get_bdev() {
+ local b d dev h i=$1 j realdev
+
+ is_number "$i" || return $?
+ echo reconfigure | multipathd -k >&/dev/null
+ for ((j=0;j<50;j++)); do
+ dev="/dev/disk/by-id/dm-uuid-mpath-$(get_bdev_uid "$i")" ||
+ continue
+ [ -e "$dev" ] && break
+ sleep .1
+ done
+ if [ ! -e "$dev" ]; then
+ echo "$dev: not found"
+ return 1
+ fi
+ if [ ! -L "$dev" ]; then
+ echo "$dev: not a soft link"
+ return 1
+ fi
+ realdev=$(readlink "$dev" 2>/dev/null || echo "?")
+ echo "Using $dev -> ${realdev}" >>"$FULL"
+ for ((j=0; j<50; j++)); do
+ blockdev --getbsz "$dev" >&/dev/null && break
+ echo reconfigure | multipathd -k >& /dev/null
+ sleep .1
+ done
+ if ! blockdev --getbsz "$dev" >&/dev/null; then
+ return 1
+ fi
+ b=$(basename "$realdev")
+ set_scheduler "$b" "${elevator}"
+ for d in /sys/class/block/*"/holders/$b"; do
+ [ -e "$d" ] || continue
+ h="$(basename "$(dirname "$(dirname "$d")")")"
+ set_scheduler "$h" "${elevator}"
+ if [ -e "/sys/class/block/$h/device/timeout" ]; then
+ echo $ini_timeout > "/sys/class/block/$h/device/timeout"
+ fi
+ done
+ echo "$dev"
+}
+
+function mountpoint() {
+ if [ -z "$TMPDIR" ]; then
+ echo "Error: \$TMPDIR has not been set." 1>&2
+ exit 1
+ fi
+ if [ -z "$1" ]; then
+ echo "Error: missing argument" 1>&2
+ exit 1
+ fi
+ echo "$TMPDIR/mnt$1"
+}
+
+all_primary_gids() {
+ find /sys/devices -name infiniband | while read -r p; do
+ cat "$p"/*/ports/*/gids/0
+ done | grep -v ':0000:0000:0000:0000$'
+}
+
+# Check whether or not an rdma_rxe instance has been associated with network
+# interface $1.
+has_rdma_rxe() {
+ local f
+
+ for f in /sys/class/infiniband/*/parent; do
+ if [ -e "$f" ] && [ "$(<"$f")" = "$1" ]; then
+ return 0
+ fi
+ done
+
+ return 1
+}
+
+# Load the rdma_rxe kernel module and associate it with all network interfaces.
+start_rdma_rxe() {
+ {
+ modprobe rdma_rxe || return $?
+ (
+ cd /sys/class/net &&
+ for i in *; do
+ if [ -e "$i" ] && ! has_rdma_rxe "$i"; then
+ echo "$i" > /sys/module/rdma_rxe/parameters/add
+ fi
+ done
+ )
+ } >>"$FULL"
+}
+
+# Dissociate the rdma_rxe kernel module from all network interfaces and unload
+# the rdma_rxe kernel module.
+stop_rdma_rxe() {
+ (
+ cd /sys/class/net &&
+ for i in *; do
+ if [ -e "$i" ] && has_rdma_rxe "$i"; then
+ { echo "$i" > /sys/module/rdma_rxe/parameters/remove; } \
+ 2>/dev/null
+ fi
+ done
+ )
+ if ! unload_module rdma_rxe; then
+ echo "Unloading rdma_rxe failed"
+ return 1
+ fi
+}
+
+scsi_debug_dev_path() {
+ local bd="" d
+
+ for d in /sys/bus/pseudo/drivers/scsi_debug/adapter*/host*/target*/*/block/*; do
+ [ -e "$d" ] || continue
+ bd=${d/*\//}
+ done
+ [ -n "$bd" ] || return 1
+ echo "/dev/$bd"
+}
+
+configure_nvmet_port() {
+ local p=$1 ipv4_addr=$2 i
+
+ echo "Configuring $p with address $ipv4_addr as an NVMeOF target port" \
+ >>"$FULL"
+ (
+ cd /sys/kernel/config/nvmet/ports &&
+ for ((i=1;1;i++)); do [ -e "$i" ] || break; done &&
+ mkdir "$i" &&
+ cd "$i" &&
+ echo ipv4 > addr_adrfam &&
+ echo rdma > addr_trtype &&
+ echo -n "$ipv4_addr" > addr_traddr &&
+ echo -n ${nvme_port} > addr_trsvcid
+ )
+}
+
+start_nvme_target() {
+ local d i ipv4_addr num_ports=0
+
+ modprobe nvme dyndbg=+pmf &&
+ modprobe nvmet-rdma dyndbg=+pmf &&
+ sleep .1 &&
+ (
+ cd /sys/kernel/config/nvmet/subsystems &&
+ mkdir ${nvme_subsysnqn} &&
+ cd ${nvme_subsysnqn} &&
+ cd namespaces &&
+ mkdir "${namespace[0]}" &&
+ cd "${namespace[0]}" &&
+ echo 00000000-0000-0000-0000-000000000000 >device_nguid &&
+ echo -n /dev/nullb0 >device_path &&
+ echo 1 >enable &&
+ cd ../.. &&
+ echo 1 >attr_allow_any_host
+ ) && for i in $(rdma_network_interfaces); do
+ ipv4_addr=$(get_ipv4_addr "$i")
+ if [ -n "${ipv4_addr}" ]; then
+ configure_nvmet_port "$i" "${ipv4_addr}"
+ ((num_ports++))
+ true
+ fi
+ done &&
+ if [ $num_ports = 0 ]; then
+ echo "No NVMeOF target ports"
+ false
+ fi && (
+ cd /sys/kernel/config/nvmet/ports &&
+ for i in *; do
+ [ -e "$i" ] && (
+ cd "$i/subsystems" &&
+ ln -s "../../../subsystems/${nvme_subsysnqn}" .
+ )
+ done
+ )
+ echo "Configured NVMe target driver"
+}
+
+stop_nvme_target() {
+ local d
+
+ (
+ cd /sys/kernel/config/nvmet 2>/dev/null &&
+ rm -f -- ports/*/subsystems/* &&
+ for d in {*/*/*/*,*/*}; do
+ [ -e "$d" ] &&
+ [ "$(basename "$(dirname "$d")")" != ana_groups ] &&
+ rmdir "$d"
+ done
+ )
+ unload_module nvmet_rdma &&
+ unload_module nvmet &&
+ unload_null_blk
+}
+
+start_target() {
+ start_rdma_rxe
+ (
+ echo "RDMA interfaces:"
+ cd /sys/class/infiniband &&
+ for i in *; do
+ [ -e "$i" ] || continue
+ for p in "$i/ports/"*; do
+ echo "$i, port $(basename "$p"): $(<"$p/gids/0")"
+ done
+ done
+ ) &>>"$FULL"
+ start_nvme_target
+}
+
+stop_target() {
+ stop_nvme_target || return $?
+ stop_rdma_rxe || return $?
+}
+
+# Look up the block device below the filesystem for directory $1.
+block_dev_of_dir() {
+ df "$1" | {
+ read -r header
+ echo "$header" >/dev/null
+ read -r blockdev rest
+ echo "$blockdev"
+ }
+}
+
+create_filesystem() {
+ local dev=$1
+
+ case "$filesystem_type" in
+ ext4)
+ mkfs.ext4 -F -O ^has_journal -q "$dev";;
+ xfs)
+ mkfs.xfs -f -q "$dev";;
+ *)
+ return 1;;
+ esac
+}
+
+is_mountpoint() {
+ [ -n "$1" ] &&
+ [ -d "$1" ] &&
+ [ "$(block_dev_of_dir "$1")" != \
+ "$(block_dev_of_dir "$(dirname "$1")")" ]
+}
+
+# Execute mount "$@" and check whether the mount command has succeeded by
+# verifying whether after mount has finished that ${$#} is a mountpoint.
+mount_and_check() {
+ local dir last
+
+ dir=$(for last; do :; done; echo "$last")
+ mount "$@"
+ if ! is_mountpoint "$dir"; then
+ echo "Error: mount $* failed"
+ return 1
+ fi
+}
+
+# Unmount the filesystem mounted at mountpoint $1. In contrast with the umount
+# command, this function does not accept a block device as argument.
+unmount_and_check() {
+ local bd m=$1 mp
+
+ if is_mountpoint "$m"; then
+ bd=$(block_dev_of_dir "$m")
+ mp=$(dev_to_mpath "$bd") 2>/dev/null
+ if [ -n "$mp" ]; then
+ dmsetup message "$mp" 0 fail_if_no_path
+ fi
+ stop_bdev_users "$bd"
+ echo "Unmounting $m from $bd" >> "$FULL"
+ umount "$m" || umount --lazy "$m"
+ fi
+ if is_mountpoint "$m"; then
+ echo "Error: unmounting $m failed"
+ return 1
+ fi
+}
+
+# Test whether fio supports command-line options "$@"
+test_fio_opt() {
+ local opt
+
+ for opt in "$@"; do
+ opt=${opt//=*}
+ fio --help |& grep -q -- "${opt}=" && continue
+ opt=${opt#--}
+ fio --cmdhelp=all |& grep -q "^${opt}[[:blank:]]" && continue
+ return 1
+ done
+}
+
+run_fio() {
+ local a args avail_kb="" bd="" d="" j opt output
+
+ args=("$@")
+ j=1
+ for opt in "${args[@]}"; do
+ case "$opt" in
+ --directory=*) d="${opt#--directory=}";;
+ --filename=*) bd="${opt#--filename=}";;
+ --numjobs=*) j="${opt#--numjobs=}";;
+ --output=*) output="${opt#--output=}";;
+ esac
+ done
+ if [ -n "$d" ]; then
+ a=$(df "$d" | grep "^/" |
+ {
+ if read -r fs blocks used avail use mnt; then
+ echo "$avail"
+ echo "$fs $blocks $used $use $mnt" >/dev/null
+ fi
+ }
+ )
+ avail_kb=$a
+ fi
+ if [ -n "$bd" ]; then
+ avail_kb=$(("$(blockdev --getsz "$bd")" / 2))
+ fi
+ if [ -n "$avail_kb" ]; then
+ args+=("--size=$(((avail_kb * 1024 * 7 / 10) / j & ~4095))")
+ fi
+ for opt in --exitall_on_error=1 --gtod_reduce=1 --aux-path=${fio_aux_path}
+ do
+ if test_fio_opt "$opt"; then
+ args+=("$opt")
+ fi
+ done
+ mkdir -p "${fio_aux_path}"
+ echo "fio ${args[*]}" >>"$FULL"
+ fio "${args[@]}" 2>&1 || return $?
+ if [ -n "$output" ]; then
+ # Return exit code 1 if no I/O has been performed.
+ grep -q ', io=[0-9].*, run=[0-9]' "$output"
+ fi
+}
+
+# Configure two null_blk instances.
+configure_null_blk() {
+ local i
+
+ modprobe null_blk nr_devices=0 || return $?
+ (
+ cd /sys/kernel/config/nullb || return $?
+ for i in nullb0 nullb1; do (
+ { mkdir -p $i &&
+ cd $i &&
+ echo 0 > completion_nsec &&
+ echo 512 > blocksize &&
+ echo $((ramdisk_size>>20)) > size &&
+ echo 1 > memory_backed &&
+ echo 1 > power; } || exit $?
+ ) done
+ )
+ ls -l /dev/nullb* &>>"$FULL"
+}
+
+unload_null_blk() {
+ local d
+
+ for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d"; done
+ unload_module null_blk
+}
+
+shutdown_client() {
+ remove_mpath_devs &&
+ log_out &&
+ stop_client
+}
+
+# Undo setup()
+teardown() {
+ killall -9 multipathd >&/dev/null
+ rm -f /etc/multipath.conf
+ stop_target
+ unload_null_blk
+}
+
+# Set up test configuration
+setup() {
+ local i m modules
+
+ set -u
+
+ shutdown_client || return $?
+
+ if ! teardown; then
+ echo "teardown() failed"
+ return 1
+ fi
+
+ modules=(
+ configfs
+ dm-multipath
+ dm_mod
+ scsi_dh_alua
+ scsi_dh_emc
+ scsi_dh_rdac
+ scsi_mod
+ )
+ for m in "${modules[@]}"; do
+ [ -e "/sys/module/$m" ] || modprobe "$m" || return $?
+ done
+
+ configure_null_blk
+
+ if [ ! -e /etc/multipath.conf ]; then
+ (
+ srcdir=$PWD
+ cd /etc && ln -s "$srcdir/tests/nvmeof-mp/multipath.conf" .
+ )
+ fi
+ multipathd
+
+ # Load the I/O scheduler kernel modules
+ (
+ cd "/lib/modules/$(uname -r)/kernel/block" &&
+ for m in *.ko; do
+ [ -e "$m" ] && modprobe "${m%.ko}"
+ done
+ )
+
+ if [ -d /sys/kernel/debug/dynamic_debug ]; then
+ for m in ; do
+ echo "module $m +pmf" >/sys/kernel/debug/dynamic_debug/control
+ done
+ fi
+
+ start_target
+}
Add a series of tests for the NVMeOF drivers on top of the dm-mpath driver. These tests are similar to the tests under tests/srp. Both tests use the dm-mpath driver for multipath and the loopback functionality of the rdma_rxe driver. The only difference is that the nvmeof-mp tests use the NVMeOF initiator and target drivers instead of the SRP initiator and target drivers. Signed-off-by: Bart Van Assche <bart.vanassche@wdc.com> --- tests/nvmeof-mp/.gitignore | 1 + tests/nvmeof-mp/001 | 50 ++ tests/nvmeof-mp/001.out | 3 + tests/nvmeof-mp/002 | 50 ++ tests/nvmeof-mp/002.out | 2 + tests/nvmeof-mp/004 | 50 ++ tests/nvmeof-mp/004.out | 2 + tests/nvmeof-mp/005 | 40 ++ tests/nvmeof-mp/005.out | 2 + tests/nvmeof-mp/006 | 40 ++ tests/nvmeof-mp/006.out | 2 + tests/nvmeof-mp/009 | 40 ++ tests/nvmeof-mp/009.out | 2 + tests/nvmeof-mp/010 | 40 ++ tests/nvmeof-mp/010.out | 2 + tests/nvmeof-mp/011 | 46 ++ tests/nvmeof-mp/011.out | 2 + tests/nvmeof-mp/012 | 53 ++ tests/nvmeof-mp/012.out | 2 + tests/nvmeof-mp/getuid_callout | 11 + tests/nvmeof-mp/multipath.conf.in | 28 + tests/nvmeof-mp/rc | 936 ++++++++++++++++++++++++++++++ 22 files changed, 1404 insertions(+) create mode 100644 tests/nvmeof-mp/.gitignore create mode 100755 tests/nvmeof-mp/001 create mode 100644 tests/nvmeof-mp/001.out create mode 100755 tests/nvmeof-mp/002 create mode 100644 tests/nvmeof-mp/002.out create mode 100755 tests/nvmeof-mp/004 create mode 100644 tests/nvmeof-mp/004.out create mode 100755 tests/nvmeof-mp/005 create mode 100644 tests/nvmeof-mp/005.out create mode 100755 tests/nvmeof-mp/006 create mode 100644 tests/nvmeof-mp/006.out create mode 100755 tests/nvmeof-mp/009 create mode 100644 tests/nvmeof-mp/009.out create mode 100755 tests/nvmeof-mp/010 create mode 100644 tests/nvmeof-mp/010.out create mode 100755 tests/nvmeof-mp/011 create mode 100644 tests/nvmeof-mp/011.out create mode 100755 tests/nvmeof-mp/012 create mode 100644 tests/nvmeof-mp/012.out create mode 100755 tests/nvmeof-mp/getuid_callout create mode 100644 tests/nvmeof-mp/multipath.conf.in create mode 100755 tests/nvmeof-mp/rc