| Message ID | 20250311024221.363421-1-lizhijian@fujitsu.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | [v6] migration: Add qtest for migration over RDMA | expand |
Hi Fabiano, A minor improvement inline below, On 11/03/2025 10:42, Li Zhijian wrote: > This qtest requires there is a RDMA(RoCE) link in the host. > In order to make the test work smoothly, introduce a > scripts/rdma-migration-helper.sh to detect existing RoCE link before > running the test. > > Test will be skipped if there is no available RoCE link. > # Start of rdma tests > # Running /x86_64/migration/precopy/rdma/plain > ok 1 /x86_64/migration/precopy/rdma/plain # SKIP No rdma link available > # To enable the test: > # Run 'scripts/rdma-migration-helper.sh setup' with root to setup a new rdma/rxe link and rerun the test > # Optional: run 'scripts/rdma-migration-helper.sh clean' to revert the 'setup' > > # End of rdma tests > > Cc: Philippe Mathieu-Daudé <philmd@linaro.org> > Cc: Stefan Hajnoczi <stefanha@gmail.com> > Reviewed-by: Peter Xu <peterx@redhat.com> > Signed-off-by: Li Zhijian <lizhijian@fujitsu.com> > --- > > Hi Fabiano, > > Please replace this patch in your new PR. > --- > V6: > - make scripts/rdma-migration-helper.sh rubost, including > 1) only setup rdma/rxe on a valid ipv4 interface # resolve gitlab-ci aarch64 runner > 2) split setup and detect operation > 3) check root for setup/clean operation > - rdma migration test only detect rdma link, local users should 'setup' rdma explicitly # don't dirty the system quietly > - print skip message in TAP format, starting with # > --- > MAINTAINERS | 1 + > scripts/rdma-migration-helper.sh | 70 +++++++++++++++++++++++++++ > tests/qtest/migration/precopy-tests.c | 66 +++++++++++++++++++++++++ > 3 files changed, 137 insertions(+) > create mode 100755 scripts/rdma-migration-helper.sh > > diff --git a/MAINTAINERS b/MAINTAINERS > index 3848d37a38..15360fcdc4 100644 > --- a/MAINTAINERS > +++ b/MAINTAINERS > @@ -3480,6 +3480,7 @@ R: Li Zhijian <lizhijian@fujitsu.com> > R: Peter Xu <peterx@redhat.com> > S: Odd Fixes > F: migration/rdma* > +F: scripts/rdma-migration-helper.sh > > Migration dirty limit and dirty page rate > M: Hyman Huang <yong.huang@smartx.com> > diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh > new file mode 100755 > index 0000000000..c4382134cd > --- /dev/null > +++ b/scripts/rdma-migration-helper.sh > @@ -0,0 +1,70 @@ > +#!/bin/bash > + > +# Copied from blktests > +get_ipv4_addr() > +{ > + ip -4 -o addr show dev "$1" | > + sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | > + tr -d '\n' this line can be improved to: head -1 | tr -d '\n' which adapts to the interface with multiple ipv4 addresses Please let me if you want a new version. Thanks Zhijian > +} > + > +# existing rdma interfaces > +rdma_interfaces() > +{ > + rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' > +} > + > +# existing valid ipv4 interfaces > +ipv4_interfaces() > +{ > + ip -o addr show | awk '/inet / {print $2}' | grep -v -w lo > +} > + > +rdma_rxe_detect() > +{ > + for r in $(rdma_interfaces) > + do > + ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return > + done > + > + return 1 > +} > + > +rdma_rxe_setup() > +{ > + for i in $(ipv4_interfaces) > + do > + rdma_interfaces | grep -qw $i && continue > + rdma link add "${i}_rxe" type rxe netdev "$i" && { > + echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)" > + return > + } > + done > + > + echo "Failed to setup any new rdma/rxe link" >&2 > + return 1 > +} > + > +rdma_rxe_clean() > +{ > + modprobe -r rdma_rxe > +} > + > +operation=${1:-detect} > + > +command -v rdma >/dev/null || { > + echo "Command 'rdma' is not available, please install it first." >&2 > + exit 1 > +} > + > +if [ "$operation" == "setup" ] || [ "$operation" == "clean" ]; then > + [ "$UID" == 0 ] || { > + echo "Root privilege is required to setup/clean a rdma/rxe link" >&2 > + exit 1 > + } > + rdma_rxe_"$operation" > +elif [ "$operation" == "detect" ]; then > + rdma_rxe_detect > +else > + echo "Usage: $0 [setup | detect | clean]" > +fi > diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c > index ba273d10b9..557b7f3aa7 100644 > --- a/tests/qtest/migration/precopy-tests.c > +++ b/tests/qtest/migration/precopy-tests.c > @@ -99,6 +99,68 @@ static void test_precopy_unix_dirty_ring(void) > test_precopy_common(&args); > } > > +#ifdef CONFIG_RDMA > + > +#define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh" > +static int new_rdma_link(char *buffer) > +{ > + char cmd[256]; > + bool verbose = g_getenv("QTEST_LOG"); > + > + snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER, > + verbose ? "" : "2>/dev/null"); > + > + FILE *pipe = popen(cmd, "r"); > + if (pipe == NULL) { > + perror("Failed to run script"); > + return -1; > + } > + > + int idx = 0; > + while (fgets(buffer + idx, 128 - idx, pipe) != NULL) { > + idx += strlen(buffer); > + } > + > + int status = pclose(pipe); > + if (status == -1) { > + perror("Error reported by pclose()"); > + return -1; > + } else if (WIFEXITED(status)) { > + return WEXITSTATUS(status); > + } > + > + return -1; > +} > + > +static void test_precopy_rdma_plain(void) > +{ > + char buffer[128] = {}; > + > + if (new_rdma_link(buffer)) { > + g_test_skip("No rdma link available\n" > + "# To enable the test:\n" > + "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup" > + " a new rdma/rxe link and rerun the test\n" > + "# Optional: run 'scripts/rdma-migration-helper.sh clean' to" > + " revert the 'setup'\n"); > + return; > + } > + > + /* > + * TODO: query a free port instead of hard code. > + * 29200=('R'+'D'+'M'+'A')*100 > + **/ > + g_autofree char *uri = g_strdup_printf("rdma:%s:29200", buffer); > + > + MigrateCommon args = { > + .listen_uri = uri, > + .connect_uri = uri, > + }; > + > + test_precopy_common(&args); > +} > +#endif > + > static void test_precopy_tcp_plain(void) > { > MigrateCommon args = { > @@ -1124,6 +1186,10 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env) > test_multifd_tcp_uri_none); > migration_test_add("/migration/multifd/tcp/plain/cancel", > test_multifd_tcp_cancel); > +#ifdef CONFIG_RDMA > + migration_test_add("/migration/precopy/rdma/plain", > + test_precopy_rdma_plain); > +#endif > } > > void migration_test_add_precopy(MigrationTestEnv *env)
diff --git a/MAINTAINERS b/MAINTAINERS index 3848d37a38..15360fcdc4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3480,6 +3480,7 @@ R: Li Zhijian <lizhijian@fujitsu.com> R: Peter Xu <peterx@redhat.com> S: Odd Fixes F: migration/rdma* +F: scripts/rdma-migration-helper.sh Migration dirty limit and dirty page rate M: Hyman Huang <yong.huang@smartx.com> diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh new file mode 100755 index 0000000000..c4382134cd --- /dev/null +++ b/scripts/rdma-migration-helper.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Copied from blktests +get_ipv4_addr() +{ + ip -4 -o addr show dev "$1" | + sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | + tr -d '\n' +} + +# existing rdma interfaces +rdma_interfaces() +{ + rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' +} + +# existing valid ipv4 interfaces +ipv4_interfaces() +{ + ip -o addr show | awk '/inet / {print $2}' | grep -v -w lo +} + +rdma_rxe_detect() +{ + for r in $(rdma_interfaces) + do + ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return + done + + return 1 +} + +rdma_rxe_setup() +{ + for i in $(ipv4_interfaces) + do + rdma_interfaces | grep -qw $i && continue + rdma link add "${i}_rxe" type rxe netdev "$i" && { + echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)" + return + } + done + + echo "Failed to setup any new rdma/rxe link" >&2 + return 1 +} + +rdma_rxe_clean() +{ + modprobe -r rdma_rxe +} + +operation=${1:-detect} + +command -v rdma >/dev/null || { + echo "Command 'rdma' is not available, please install it first." >&2 + exit 1 +} + +if [ "$operation" == "setup" ] || [ "$operation" == "clean" ]; then + [ "$UID" == 0 ] || { + echo "Root privilege is required to setup/clean a rdma/rxe link" >&2 + exit 1 + } + rdma_rxe_"$operation" +elif [ "$operation" == "detect" ]; then + rdma_rxe_detect +else + echo "Usage: $0 [setup | detect | clean]" +fi diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c index ba273d10b9..557b7f3aa7 100644 --- a/tests/qtest/migration/precopy-tests.c +++ b/tests/qtest/migration/precopy-tests.c @@ -99,6 +99,68 @@ static void test_precopy_unix_dirty_ring(void) test_precopy_common(&args); } +#ifdef CONFIG_RDMA + +#define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh" +static int new_rdma_link(char *buffer) +{ + char cmd[256]; + bool verbose = g_getenv("QTEST_LOG"); + + snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER, + verbose ? "" : "2>/dev/null"); + + FILE *pipe = popen(cmd, "r"); + if (pipe == NULL) { + perror("Failed to run script"); + return -1; + } + + int idx = 0; + while (fgets(buffer + idx, 128 - idx, pipe) != NULL) { + idx += strlen(buffer); + } + + int status = pclose(pipe); + if (status == -1) { + perror("Error reported by pclose()"); + return -1; + } else if (WIFEXITED(status)) { + return WEXITSTATUS(status); + } + + return -1; +} + +static void test_precopy_rdma_plain(void) +{ + char buffer[128] = {}; + + if (new_rdma_link(buffer)) { + g_test_skip("No rdma link available\n" + "# To enable the test:\n" + "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup" + " a new rdma/rxe link and rerun the test\n" + "# Optional: run 'scripts/rdma-migration-helper.sh clean' to" + " revert the 'setup'\n"); + return; + } + + /* + * TODO: query a free port instead of hard code. + * 29200=('R'+'D'+'M'+'A')*100 + **/ + g_autofree char *uri = g_strdup_printf("rdma:%s:29200", buffer); + + MigrateCommon args = { + .listen_uri = uri, + .connect_uri = uri, + }; + + test_precopy_common(&args); +} +#endif + static void test_precopy_tcp_plain(void) { MigrateCommon args = { @@ -1124,6 +1186,10 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env) test_multifd_tcp_uri_none); migration_test_add("/migration/multifd/tcp/plain/cancel", test_multifd_tcp_cancel); +#ifdef CONFIG_RDMA + migration_test_add("/migration/precopy/rdma/plain", + test_precopy_rdma_plain); +#endif } void migration_test_add_precopy(MigrationTestEnv *env)