[v4] Prevent vhost-user-blk-test hang

Commit Message

Raphael Norwitz Sept. 23, 2021, 3:16 p.m. UTC

In the vhost-user-blk-test, as of now there is nothing stoping
vhost-user-blk in QEMU writing to the socket right after forking off the
storage daemon before it has a chance to come up properly, leaving the
test hanging forever. This intermittently hanging test has caused QEMU
automation failures reported multiple times on the mailing list [1].

This change makes the storage-daemon notify the vhost-user-blk-test
that it is fully initialized and ready to handle client connections by
creating a pidfile on initialiation. This ensures that the storage-daemon
backend won't miss vhost-user messages and thereby resolves the hang.

[1] https://lore.kernel.org/qemu-devel/CAFEAcA8kYpz9LiPNxnWJAPSjc=[e-mail]/

Signed-off-by: Raphael Norwitz <[e-mail]>
---
Accidentally left a raw free() in v3. Converted it to g_free() here.

 tests/qtest/vhost-user-blk-test.c | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

Comments

Eric Blake Sept. 23, 2021, 9:22 p.m. UTC | #1

On Thu, Sep 23, 2021 at 03:16:17PM +0000, Raphael Norwitz wrote:
> In the vhost-user-blk-test, as of now there is nothing stoping
> vhost-user-blk in QEMU writing to the socket right after forking off the
> storage daemon before it has a chance to come up properly, leaving the
> test hanging forever. This intermittently hanging test has caused QEMU
> automation failures reported multiple times on the mailing list [1].
> 
> This change makes the storage-daemon notify the vhost-user-blk-test
> that it is fully initialized and ready to handle client connections by
> creating a pidfile on initialiation. This ensures that the storage-daemon
> backend won't miss vhost-user messages and thereby resolves the hang.
> 
> [1] https://lore.kernel.org/qemu-devel/CAFEAcA8kYpz9LiPNxnWJAPSjc=[e-mail]/
> 
> Signed-off-by: Raphael Norwitz <[e-mail]>
> ---
> Accidentally left a raw free() in v3. Converted it to g_free() here.
> 
>  tests/qtest/vhost-user-blk-test.c | 27 ++++++++++++++++++++++++++-
>  1 file changed, 26 insertions(+), 1 deletion(-)
> 
> diff --git a/tests/qtest/vhost-user-blk-test.c b/tests/qtest/vhost-user-blk-test.c
> index 6f108a1b62..6898f55f11 100644
> --- a/tests/qtest/vhost-user-blk-test.c
> +++ b/tests/qtest/vhost-user-blk-test.c
> @@ -24,6 +24,7 @@
>  #define TEST_IMAGE_SIZE         (64 * 1024 * 1024)
>  #define QVIRTIO_BLK_TIMEOUT_US  (30 * 1000 * 1000)
>  #define PCI_SLOT_HP             0x06
> +#define PIDFILE_RETRIES         5
>  
>  typedef struct {
>      pid_t pid;
> @@ -885,7 +886,8 @@ static void start_vhost_user_blk(GString *cmd_line, int vus_instances,
>                                   int num_queues)
>  {
>      const char *vhost_user_blk_bin = qtest_qemu_storage_daemon_binary();
> -    int i;
> +    int i, retries;
> +    char *daemon_pidfile_path;
>      gchar *img_path;
>      GString *storage_daemon_command = g_string_new(NULL);
>      QemuStorageDaemonState *qsd;
> @@ -898,6 +900,9 @@ static void start_vhost_user_blk(GString *cmd_line, int vus_instances,
>              " -object memory-backend-memfd,id=mem,size=256M,share=on "
>              " -M memory-backend=mem -m 256M ");
>  
> +    daemon_pidfile_path = g_strdup_printf("/tmp/daemon-%d", getpid());
> +    g_assert_cmpint((uintptr_t) daemon_pidfile_path, !=, (uintptr_t) NULL);

Feels verbose compared to:
g_assert(daemon_pidfile_path)

For that matter, can g_strdup_printf() ever return NULL?  On memory
error, it exit()s rather than returning.  So this assert feels like
overkill.

Otherwise,

Reviewed-by: Eric Blake <[e-mail]>

Patch

diff --git a/tests/qtest/vhost-user-blk-test.c b/tests/qtest/vhost-user-blk-test.c
index 6f108a1b62..6898f55f11 100644
--- a/tests/qtest/vhost-user-blk-test.c
+++ b/tests/qtest/vhost-user-blk-test.c
@@ -24,6 +24,7 @@ 
 #define TEST_IMAGE_SIZE         (64 * 1024 * 1024)
 #define QVIRTIO_BLK_TIMEOUT_US  (30 * 1000 * 1000)
 #define PCI_SLOT_HP             0x06
+#define PIDFILE_RETRIES         5
 
 typedef struct {
     pid_t pid;
@@ -885,7 +886,8 @@  static void start_vhost_user_blk(GString *cmd_line, int vus_instances,
                                  int num_queues)
 {
     const char *vhost_user_blk_bin = qtest_qemu_storage_daemon_binary();
-    int i;
+    int i, retries;
+    char *daemon_pidfile_path;
     gchar *img_path;
     GString *storage_daemon_command = g_string_new(NULL);
     QemuStorageDaemonState *qsd;
@@ -898,6 +900,9 @@  static void start_vhost_user_blk(GString *cmd_line, int vus_instances,
             " -object memory-backend-memfd,id=mem,size=256M,share=on "
             " -M memory-backend=mem -m 256M ");
 
+    daemon_pidfile_path = g_strdup_printf("/tmp/daemon-%d", getpid());
+    g_assert_cmpint((uintptr_t) daemon_pidfile_path, !=, (uintptr_t) NULL);
+
     for (i = 0; i < vus_instances; i++) {
         int fd;
         char *sock_path = create_listen_socket(&fd);
@@ -914,6 +919,9 @@  static void start_vhost_user_blk(GString *cmd_line, int vus_instances,
                                i + 1, sock_path);
     }
 
+    g_string_append_printf(storage_daemon_command, "--pidfile %s ",
+                           daemon_pidfile_path);
+
     g_test_message("starting vhost-user backend: %s",
                    storage_daemon_command->str);
     pid_t pid = fork();
@@ -930,7 +938,24 @@  static void start_vhost_user_blk(GString *cmd_line, int vus_instances,
         execlp("/bin/sh", "sh", "-c", storage_daemon_command->str, NULL);
         exit(1);
     }
+
+    /*
+     * Ensure the storage-daemon has come up properly before allowing the
+     * test to proceed.
+     */
+    retries = 0;
+    while (access(daemon_pidfile_path, F_OK) != 0) {
+        g_assert_cmpint(retries, <, PIDFILE_RETRIES);
+
+        retries++;
+        g_usleep(1000);
+    }
+
     g_string_free(storage_daemon_command, true);
+    if (access(daemon_pidfile_path, F_OK) == 0) {
+        unlink(daemon_pidfile_path);
+    }
+    g_free(daemon_pidfile_path);
 
     qsd = g_new(QemuStorageDaemonState, 1);
     qsd->pid = pid;