diff mbox

[12/13] migration: [HACK]Transfer pages over new channels

Message ID 1477078935-7182-13-git-send-email-quintela@redhat.com (mailing list archive)
State New, archived
Headers show

Commit Message

Juan Quintela Oct. 21, 2016, 7:42 p.m. UTC 
We switch for sending the page number to send real pages.

[HACK]
How we calculate the bandwidth is beyond repair, there is a hack there
that would work for x86 and archs that have 4kb pages.

If you are having a nice day just go to migration/ram.c and look at
acct_update_position().  Now you are depressed, right?

Signed-off-by: Juan Quintela <quintela@redhat.com>
---
 migration/migration.c | 15 +++++++++++----
 migration/ram.c       | 46 +++++++++++++++++++++++++++++++---------------
 2 files changed, 42 insertions(+), 19 deletions(-)

Comments

Dr. David Alan Gilbert Oct. 26, 2016, 7:08 p.m. UTC | #1 
* Juan Quintela (quintela@redhat.com) wrote:
> We switch for sending the page number to send real pages.
> 
> [HACK]
> How we calculate the bandwidth is beyond repair, there is a hack there
> that would work for x86 and archs that have 4kb pages.

Is that a problem? Isn't it always TARGET_PAGE_SIZE?
But it does add an interesting question about what size chunks
to hand out to each fd, do you bother with 4kB chunks or go for something
larger?

> If you are having a nice day just go to migration/ram.c and look at
> acct_update_position().  Now you are depressed, right?

Been there, then saw the interaction with RDMA.

Dave

> Signed-off-by: Juan Quintela <quintela@redhat.com>
> ---
>  migration/migration.c | 15 +++++++++++----
>  migration/ram.c       | 46 +++++++++++++++++++++++++++++++---------------
>  2 files changed, 42 insertions(+), 19 deletions(-)
> 
> diff --git a/migration/migration.c b/migration/migration.c
> index 407e0c3..0627f14 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -1757,7 +1757,8 @@ static void *migration_thread(void *opaque)
>      /* Used by the bandwidth calcs, updated later */
>      int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
>      int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
> -    int64_t initial_bytes = 0;
> +    int64_t qemu_file_bytes = 0;
> +    int64_t multifd_pages = 0;
>      int64_t max_size = 0;
>      int64_t start_time = initial_time;
>      int64_t end_time;
> @@ -1840,9 +1841,14 @@ static void *migration_thread(void *opaque)
>          }
>          current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
>          if (current_time >= initial_time + BUFFER_DELAY) {
> -            uint64_t transferred_bytes = qemu_ftell(s->to_dst_file) -
> -                                         initial_bytes;
>              uint64_t time_spent = current_time - initial_time;
> +            uint64_t qemu_file_bytes_now = qemu_ftell(s->to_dst_file);
> +            uint64_t multifd_pages_now = multifd_mig_pages_transferred();
> +            /* Hack ahead.  Why the hell we don't have a function to now the
> +               target_page_size.  Hard coding it to 4096 */
> +            uint64_t transferred_bytes =
> +                (qemu_file_bytes_now - qemu_file_bytes) +
> +                (multifd_pages_now - multifd_pages) * 4096;
>              double bandwidth = (double)transferred_bytes / time_spent;
>              max_size = bandwidth * s->parameters.downtime_limit;
> 
> @@ -1859,7 +1865,8 @@ static void *migration_thread(void *opaque)
> 
>              qemu_file_reset_rate_limit(s->to_dst_file);
>              initial_time = current_time;
> -            initial_bytes = qemu_ftell(s->to_dst_file);
> +            qemu_file_bytes = qemu_file_bytes_now;
> +            multifd_pages = multifd_pages_now;
>          }
>          if (qemu_file_rate_limit(s->to_dst_file)) {
>              /* usleep expects microseconds */
> diff --git a/migration/ram.c b/migration/ram.c
> index 2ead443..9a20f63 100644
> --- a/migration/ram.c
> +++ b/migration/ram.c
> @@ -437,9 +437,9 @@ static void *multifd_send_thread(void *opaque)
>              params->address = 0;
>              qemu_mutex_unlock(&params->mutex);
> 
> -            if (qio_channel_write(params->c, (const char *)&address,
> -                                  sizeof(uint8_t *), &error_abort)
> -                != sizeof(uint8_t*)) {
> +            if (qio_channel_write(params->c, (const char *)address,
> +                                  TARGET_PAGE_SIZE, &error_abort)
> +                != TARGET_PAGE_SIZE) {
>                  /* Shuoudn't ever happen */
>                  exit(-1);
>              }
> @@ -551,6 +551,23 @@ static int multifd_send_page(uint8_t *address)
>      return i;
>  }
> 
> +static void flush_multifd_send_data(QEMUFile *f)
> +{
> +    int i, thread_count;
> +
> +    if (!migrate_multifd()) {
> +        return;
> +    }
> +    qemu_fflush(f);
> +    thread_count = migrate_multifd_threads();
> +    qemu_mutex_lock(&multifd_send_mutex);
> +    for (i = 0; i < thread_count; i++) {
> +        while(!multifd_send[i].done) {
> +            qemu_cond_wait(&multifd_send_cond, &multifd_send_mutex);
> +        }
> +    }
> +}
> +
>  struct MultiFDRecvParams {
>      /* not changed */
>      QemuThread thread;
> @@ -575,7 +592,6 @@ static void *multifd_recv_thread(void *opaque)
>  {
>      MultiFDRecvParams *params = opaque;
>      uint8_t *address;
> -    uint8_t *recv_address;
>      char start;
> 
>      qio_channel_read(params->c, &start, 1, &error_abort);
> @@ -591,19 +607,13 @@ static void *multifd_recv_thread(void *opaque)
>              params->address = 0;
>              qemu_mutex_unlock(&params->mutex);
> 
> -            if (qio_channel_read(params->c, (char *)&recv_address,
> -                                 sizeof(uint8_t*), &error_abort)
> -                != sizeof(uint8_t *)) {
> +            if (qio_channel_read(params->c, (char *)address,
> +                                 TARGET_PAGE_SIZE, &error_abort)
> +                != TARGET_PAGE_SIZE) {
>                  /* shouldn't ever happen */
>                  exit(-1);
>              }
> 
> -            if (address != recv_address) {
> -                printf("We received %p what we were expecting %p\n",
> -                       recv_address, address);
> -                exit(-1);
> -            }
> -
>              qemu_mutex_lock(&multifd_recv_mutex);
>              params->done = true;
>              qemu_cond_signal(&multifd_recv_cond);
> @@ -1126,6 +1136,7 @@ static int ram_multifd_page(QEMUFile *f, PageSearchStatus *pss,
>      uint8_t *p;
>      RAMBlock *block = pss->block;
>      ram_addr_t offset = pss->offset;
> +    static int count = 32;
> 
>      p = block->host + offset;
> 
> @@ -1137,9 +1148,14 @@ static int ram_multifd_page(QEMUFile *f, PageSearchStatus *pss,
>          *bytes_transferred +=
>              save_page_header(f, block, offset | RAM_SAVE_FLAG_MULTIFD_PAGE);
>          fd_num = multifd_send_page(p);
> +        count--;
> +        if (!count) {
> +            qemu_fflush(f);
> +            count = 32;
> +        }
> +
>          qemu_put_be16(f, fd_num);
>          *bytes_transferred += 2; /* size of fd_num */
> -        qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
>          *bytes_transferred += TARGET_PAGE_SIZE;
>          pages = 1;
>          acct_info.norm_pages++;
> @@ -2401,6 +2417,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
>      }
> 
>      flush_compressed_data(f);
> +    flush_multifd_send_data(f);
>      ram_control_after_iterate(f, RAM_CONTROL_FINISH);
> 
>      rcu_read_unlock();
> @@ -2915,7 +2932,6 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
>          case RAM_SAVE_FLAG_MULTIFD_PAGE:
>              fd_num = qemu_get_be16(f);
>              multifd_recv_page(host, fd_num);
> -            qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
>              break;
> 
>          case RAM_SAVE_FLAG_EOS:
> -- 
> 2.7.4
> 
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
diff mbox

Patch

diff --git a/migration/migration.c b/migration/migration.c
index 407e0c3..0627f14 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1757,7 +1757,8 @@  static void *migration_thread(void *opaque)
     /* Used by the bandwidth calcs, updated later */
     int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
-    int64_t initial_bytes = 0;
+    int64_t qemu_file_bytes = 0;
+    int64_t multifd_pages = 0;
     int64_t max_size = 0;
     int64_t start_time = initial_time;
     int64_t end_time;
@@ -1840,9 +1841,14 @@  static void *migration_thread(void *opaque)
         }
         current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
         if (current_time >= initial_time + BUFFER_DELAY) {
-            uint64_t transferred_bytes = qemu_ftell(s->to_dst_file) -
-                                         initial_bytes;
             uint64_t time_spent = current_time - initial_time;
+            uint64_t qemu_file_bytes_now = qemu_ftell(s->to_dst_file);
+            uint64_t multifd_pages_now = multifd_mig_pages_transferred();
+            /* Hack ahead.  Why the hell we don't have a function to now the
+               target_page_size.  Hard coding it to 4096 */
+            uint64_t transferred_bytes =
+                (qemu_file_bytes_now - qemu_file_bytes) +
+                (multifd_pages_now - multifd_pages) * 4096;
             double bandwidth = (double)transferred_bytes / time_spent;
             max_size = bandwidth * s->parameters.downtime_limit;

@@ -1859,7 +1865,8 @@  static void *migration_thread(void *opaque)

             qemu_file_reset_rate_limit(s->to_dst_file);
             initial_time = current_time;
-            initial_bytes = qemu_ftell(s->to_dst_file);
+            qemu_file_bytes = qemu_file_bytes_now;
+            multifd_pages = multifd_pages_now;
         }
         if (qemu_file_rate_limit(s->to_dst_file)) {
             /* usleep expects microseconds */
diff --git a/migration/ram.c b/migration/ram.c
index 2ead443..9a20f63 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -437,9 +437,9 @@  static void *multifd_send_thread(void *opaque)
             params->address = 0;
             qemu_mutex_unlock(&params->mutex);

-            if (qio_channel_write(params->c, (const char *)&address,
-                                  sizeof(uint8_t *), &error_abort)
-                != sizeof(uint8_t*)) {
+            if (qio_channel_write(params->c, (const char *)address,
+                                  TARGET_PAGE_SIZE, &error_abort)
+                != TARGET_PAGE_SIZE) {
                 /* Shuoudn't ever happen */
                 exit(-1);
             }
@@ -551,6 +551,23 @@  static int multifd_send_page(uint8_t *address)
     return i;
 }

+static void flush_multifd_send_data(QEMUFile *f)
+{
+    int i, thread_count;
+
+    if (!migrate_multifd()) {
+        return;
+    }
+    qemu_fflush(f);
+    thread_count = migrate_multifd_threads();
+    qemu_mutex_lock(&multifd_send_mutex);
+    for (i = 0; i < thread_count; i++) {
+        while(!multifd_send[i].done) {
+            qemu_cond_wait(&multifd_send_cond, &multifd_send_mutex);
+        }
+    }
+}
+
 struct MultiFDRecvParams {
     /* not changed */
     QemuThread thread;
@@ -575,7 +592,6 @@  static void *multifd_recv_thread(void *opaque)
 {
     MultiFDRecvParams *params = opaque;
     uint8_t *address;
-    uint8_t *recv_address;
     char start;

     qio_channel_read(params->c, &start, 1, &error_abort);
@@ -591,19 +607,13 @@  static void *multifd_recv_thread(void *opaque)
             params->address = 0;
             qemu_mutex_unlock(&params->mutex);

-            if (qio_channel_read(params->c, (char *)&recv_address,
-                                 sizeof(uint8_t*), &error_abort)
-                != sizeof(uint8_t *)) {
+            if (qio_channel_read(params->c, (char *)address,
+                                 TARGET_PAGE_SIZE, &error_abort)
+                != TARGET_PAGE_SIZE) {
                 /* shouldn't ever happen */
                 exit(-1);
             }

-            if (address != recv_address) {
-                printf("We received %p what we were expecting %p\n",
-                       recv_address, address);
-                exit(-1);
-            }
-
             qemu_mutex_lock(&multifd_recv_mutex);
             params->done = true;
             qemu_cond_signal(&multifd_recv_cond);
@@ -1126,6 +1136,7 @@  static int ram_multifd_page(QEMUFile *f, PageSearchStatus *pss,
     uint8_t *p;
     RAMBlock *block = pss->block;
     ram_addr_t offset = pss->offset;
+    static int count = 32;

     p = block->host + offset;

@@ -1137,9 +1148,14 @@  static int ram_multifd_page(QEMUFile *f, PageSearchStatus *pss,
         *bytes_transferred +=
             save_page_header(f, block, offset | RAM_SAVE_FLAG_MULTIFD_PAGE);
         fd_num = multifd_send_page(p);
+        count--;
+        if (!count) {
+            qemu_fflush(f);
+            count = 32;
+        }
+
         qemu_put_be16(f, fd_num);
         *bytes_transferred += 2; /* size of fd_num */
-        qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
         *bytes_transferred += TARGET_PAGE_SIZE;
         pages = 1;
         acct_info.norm_pages++;
@@ -2401,6 +2417,7 @@  static int ram_save_complete(QEMUFile *f, void *opaque)
     }

     flush_compressed_data(f);
+    flush_multifd_send_data(f);
     ram_control_after_iterate(f, RAM_CONTROL_FINISH);

     rcu_read_unlock();
@@ -2915,7 +2932,6 @@  static int ram_load(QEMUFile *f, void *opaque, int version_id)
         case RAM_SAVE_FLAG_MULTIFD_PAGE:
             fd_num = qemu_get_be16(f);
             multifd_recv_page(host, fd_num);
-            qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
             break;

         case RAM_SAVE_FLAG_EOS: