@@ -57,6 +57,10 @@
/* The maximum number of page_bmap that can be allocated. */
#define VIRTIO_BALLOON_PAGE_BMAP_MAX_NUM 32
+/* Types of pages to chunk */
+#define PAGE_CHUNK_TYPE_BALLOON 0 /* Chunk of inflate/deflate pages */
+#define PAGE_CHUNK_TYPE_UNUSED 1 /* Chunk of unused pages */
+
static int oom_pages = OOM_VBALLOON_DEFAULT_PAGES;
module_param(oom_pages, int, S_IRUSR | S_IWUSR);
MODULE_PARM_DESC(oom_pages, "pages to free on OOM");
@@ -67,16 +71,17 @@ static struct vfsmount *balloon_mnt;
/* Maximum number of page chunks */
#define VIRTIO_BALLOON_MAX_PAGE_CHUNKS ((8 * PAGE_SIZE - \
- sizeof(struct virtio_balloon_page_chunk)) / \
- sizeof(struct virtio_balloon_page_chunk_entry))
+ sizeof(struct virtio_balloon_miscq_msg)) / \
+ sizeof(struct virtio_balloon_page_chunk_entry))
struct virtio_balloon {
struct virtio_device *vdev;
- struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
+ struct virtqueue *inflate_vq, *deflate_vq, *stats_vq, *miscq;
/* The balloon servicing is delegated to a freezable workqueue. */
struct work_struct update_balloon_stats_work;
struct work_struct update_balloon_size_work;
+ struct work_struct miscq_handle_work;
/* Prevent updating balloon when it is being canceled. */
spinlock_t stop_update_lock;
@@ -98,6 +103,9 @@ struct virtio_balloon {
/* Synchronize access/update to this struct virtio_balloon elements */
struct mutex balloon_lock;
+ /* Miscq msg buffer for the REPORT_UNUSED_PAGES cmd */
+ struct virtio_balloon_miscq_msg *miscq_msg_rup;
+
/* Buffer for chunks of ballooned pages. */
struct virtio_balloon_page_chunk *balloon_page_chunk;
@@ -200,38 +208,85 @@ static void clear_page_bmap(struct virtio_balloon *vb,
memset(vb->page_bmap[i], 0, VIRTIO_BALLOON_PAGE_BMAP_SIZE);
}
-static void send_page_chunks(struct virtio_balloon *vb, struct virtqueue *vq)
+static void send_page_chunks(struct virtio_balloon *vb, struct virtqueue *vq,
+ int type, bool busy_wait)
{
struct scatterlist sg;
struct virtio_balloon_page_chunk *chunk;
- unsigned int len;
+ void *msg_buf;
+ unsigned int msg_len;
+ uint64_t chunk_num = 0;
+
+ switch (type) {
+ case PAGE_CHUNK_TYPE_BALLOON:
+ chunk = vb->balloon_page_chunk;
+ chunk_num = le64_to_cpu(chunk->chunk_num);
+ msg_buf = vb->balloon_page_chunk;
+ msg_len = sizeof(struct virtio_balloon_page_chunk) +
+ sizeof(struct virtio_balloon_page_chunk_entry) *
+ chunk_num;
+ break;
+ case PAGE_CHUNK_TYPE_UNUSED:
+ chunk = &vb->miscq_msg_rup->payload.chunk;
+ chunk_num = le64_to_cpu(chunk->chunk_num);
+ msg_buf = vb->miscq_msg_rup;
+ msg_len = sizeof(struct virtio_balloon_miscq_msg) +
+ sizeof(struct virtio_balloon_page_chunk_entry) *
+ chunk_num;
+ break;
+ default:
+ dev_warn(&vb->vdev->dev, "%s: chunk %d of unknown pages\n",
+ __func__, type);
+ return;
+ }
- chunk = vb->balloon_page_chunk;
- len = sizeof(__le64) +
- le64_to_cpu(chunk->chunk_num) *
- sizeof(struct virtio_balloon_page_chunk_entry);
- sg_init_one(&sg, chunk, len);
+ sg_init_one(&sg, msg_buf, msg_len);
if (!virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL)) {
virtqueue_kick(vq);
- wait_event(vb->acked, virtqueue_get_buf(vq, &len));
+ if (busy_wait)
+ while (!virtqueue_get_buf(vq, &msg_len) &&
+ !virtqueue_is_broken(vq))
+ cpu_relax();
+ else
+ wait_event(vb->acked, virtqueue_get_buf(vq, &msg_len));
+ /*
+ * Now, the chunks have been delivered to the host.
+ * Reset the filed in the structure that records the number of
+ * added chunks, so that new added chunks can be re-counted.
+ */
chunk->chunk_num = 0;
}
}
/* Add a chunk entry to the buffer. */
static void add_one_chunk(struct virtio_balloon *vb, struct virtqueue *vq,
- u64 base, u64 size)
+ int type, u64 base, u64 size)
{
- struct virtio_balloon_page_chunk *chunk = vb->balloon_page_chunk;
+ struct virtio_balloon_page_chunk *chunk;
struct virtio_balloon_page_chunk_entry *entry;
- uint64_t chunk_num = le64_to_cpu(chunk->chunk_num);
-
+ uint64_t chunk_num;
+
+ switch (type) {
+ case PAGE_CHUNK_TYPE_BALLOON:
+ chunk = vb->balloon_page_chunk;
+ chunk_num = le64_to_cpu(vb->balloon_page_chunk->chunk_num);
+ break;
+ case PAGE_CHUNK_TYPE_UNUSED:
+ chunk = &vb->miscq_msg_rup->payload.chunk;
+ chunk_num =
+ le64_to_cpu(vb->miscq_msg_rup->payload.chunk.chunk_num);
+ break;
+ default:
+ dev_warn(&vb->vdev->dev, "%s: chunk %d of unknown pages\n",
+ __func__, type);
+ return;
+ }
entry = &chunk->entry[chunk_num];
entry->base = cpu_to_le64(base << VIRTIO_BALLOON_CHUNK_BASE_SHIFT);
entry->size = cpu_to_le64(size << VIRTIO_BALLOON_CHUNK_SIZE_SHIFT);
chunk->chunk_num = cpu_to_le64(++chunk_num);
if (chunk_num == VIRTIO_BALLOON_MAX_PAGE_CHUNKS)
- send_page_chunks(vb, vq);
+ send_page_chunks(vb, vq, type, 0);
}
static void convert_bmap_to_chunks(struct virtio_balloon *vb,
@@ -259,8 +314,8 @@ static void convert_bmap_to_chunks(struct virtio_balloon *vb,
chunk_size = (next_zero - next_one) *
VIRTIO_BALLOON_PAGES_PER_PAGE;
if (chunk_size) {
- add_one_chunk(vb, vq, pfn_start + next_one,
- chunk_size);
+ add_one_chunk(vb, vq, PAGE_CHUNK_TYPE_BALLOON,
+ pfn_start + next_one, chunk_size);
pos += next_zero + 1;
}
}
@@ -305,7 +360,7 @@ static void tell_host_from_page_bmap(struct virtio_balloon *vb,
pfn_num);
}
if (le64_to_cpu(vb->balloon_page_chunk->chunk_num) > 0)
- send_page_chunks(vb, vq);
+ send_page_chunks(vb, vq, PAGE_CHUNK_TYPE_BALLOON, 0);
}
static void set_page_pfns(struct virtio_balloon *vb,
@@ -679,43 +734,186 @@ static void update_balloon_size_func(struct work_struct *work)
queue_work(system_freezable_wq, work);
}
+/* Add a message buffer for the host to fill in a request */
+static void miscq_msg_inbuf_add(struct virtio_balloon *vb,
+ struct virtio_balloon_miscq_msg *req_buf)
+{
+ struct scatterlist sg_in;
+
+ sg_init_one(&sg_in, req_buf, sizeof(struct virtio_balloon_miscq_msg));
+ if (virtqueue_add_inbuf(vb->miscq, &sg_in, 1, req_buf, GFP_KERNEL)
+ < 0) {
+ __virtio_clear_bit(vb->vdev,
+ VIRTIO_BALLOON_F_MISC_VQ);
+ dev_warn(&vb->vdev->dev, "%s: add miscq msg buf err\n",
+ __func__);
+ return;
+ }
+ virtqueue_kick(vb->miscq);
+}
+
+static void miscq_report_unused_pages(struct virtio_balloon *vb)
+{
+ struct virtio_balloon_miscq_msg *msg = vb->miscq_msg_rup;
+ struct virtqueue *vq = vb->miscq;
+ int ret = 0;
+ unsigned int order = 0, migratetype = 0;
+ struct zone *zone = NULL;
+ struct page *page = NULL;
+ u64 pfn;
+
+ msg->cmd = cpu_to_le32(VIRTIO_BALLOON_MISCQ_CMD_REPORT_UNUSED_PAGES);
+ msg->flags = 0;
+
+ for_each_populated_zone(zone) {
+ for (order = MAX_ORDER - 1; order > 0; order--) {
+ for (migratetype = 0; migratetype < MIGRATE_TYPES;
+ migratetype++) {
+ do {
+ ret = report_unused_page_block(zone,
+ order, migratetype, &page);
+ if (!ret) {
+ pfn = (u64)page_to_pfn(page);
+ add_one_chunk(vb, vq,
+ PAGE_CHUNK_TYPE_UNUSED,
+ pfn,
+ (u64)(1 << order) *
+ VIRTIO_BALLOON_PAGES_PER_PAGE);
+ }
+ } while (!ret);
+ }
+ }
+ }
+ /* Set the cmd completion flag */
+ msg->flags |= cpu_to_le32(VIRTIO_BALLOON_MISCQ_F_COMPLETION);
+ send_page_chunks(vb, vq, PAGE_CHUNK_TYPE_UNUSED, true);
+}
+
+static void miscq_handle_func(struct work_struct *work)
+{
+ struct virtio_balloon *vb;
+ struct virtio_balloon_miscq_msg *msg;
+ unsigned int len;
+
+ vb = container_of(work, struct virtio_balloon,
+ miscq_handle_work);
+ msg = virtqueue_get_buf(vb->miscq, &len);
+ if (!msg || len != sizeof(struct virtio_balloon_miscq_msg)) {
+ dev_warn(&vb->vdev->dev, "%s: invalid miscq msg len\n",
+ __func__);
+ miscq_msg_inbuf_add(vb, vb->miscq_msg_rup);
+ return;
+ }
+ switch (msg->cmd) {
+ case VIRTIO_BALLOON_MISCQ_CMD_REPORT_UNUSED_PAGES:
+ miscq_report_unused_pages(vb);
+ break;
+ default:
+ dev_warn(&vb->vdev->dev, "%s: miscq cmd %d not supported\n",
+ __func__, msg->cmd);
+ }
+ miscq_msg_inbuf_add(vb, vb->miscq_msg_rup);
+}
+
+static void miscq_request(struct virtqueue *vq)
+{
+ struct virtio_balloon *vb = vq->vdev->priv;
+
+ queue_work(system_freezable_wq, &vb->miscq_handle_work);
+}
+
static int init_vqs(struct virtio_balloon *vb)
{
- struct virtqueue *vqs[3];
- vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
- static const char * const names[] = { "inflate", "deflate", "stats" };
- int err, nvqs;
+ struct virtqueue **vqs;
+ vq_callback_t **callbacks;
+ const char **names;
+ int err = -ENOMEM;
+ int i, nvqs;
+
+ /* Inflateq and deflateq are used unconditionally */
+ nvqs = 2;
+
+ if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ))
+ nvqs++;
+ if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_MISC_VQ))
+ nvqs++;
+
+ /* Allocate space for find_vqs parameters */
+ vqs = kcalloc(nvqs, sizeof(*vqs), GFP_KERNEL);
+ if (!vqs)
+ goto err_vq;
+ callbacks = kmalloc_array(nvqs, sizeof(*callbacks), GFP_KERNEL);
+ if (!callbacks)
+ goto err_callback;
+ names = kmalloc_array(nvqs, sizeof(*names), GFP_KERNEL);
+ if (!names)
+ goto err_names;
+
+ callbacks[0] = balloon_ack;
+ names[0] = "inflate";
+ callbacks[1] = balloon_ack;
+ names[1] = "deflate";
+
+ i = 2;
+ if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
+ callbacks[i] = stats_request;
+ names[i] = "stats";
+ i++;
+ }
- /*
- * We expect two virtqueues: inflate and deflate, and
- * optionally stat.
- */
- nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
- err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names,
- NULL);
+ if (virtio_has_feature(vb->vdev,
+ VIRTIO_BALLOON_F_MISC_VQ)) {
+ callbacks[i] = miscq_request;
+ names[i] = "miscq";
+ }
+
+ err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks,
+ names, NULL);
if (err)
- return err;
+ goto err_find;
vb->inflate_vq = vqs[0];
vb->deflate_vq = vqs[1];
+ i = 2;
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
struct scatterlist sg;
- unsigned int num_stats;
- vb->stats_vq = vqs[2];
+ vb->stats_vq = vqs[i++];
/*
* Prime this virtqueue with one buffer so the hypervisor can
* use it to signal us later (it can't be broken yet!).
*/
- num_stats = update_balloon_stats(vb);
-
- sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
+ sg_init_one(&sg, vb->stats, sizeof(vb->stats));
if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
< 0)
BUG();
virtqueue_kick(vb->stats_vq);
}
+
+ if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_MISC_VQ)) {
+ vb->miscq = vqs[i];
+ /*
+ * Add the msg buf for the REPORT_UNUSED_PAGES request.
+ * The request is handled one in-flight each time. So, just
+ * use the response buffer, msicq_msg_rup, for the host to
+ * fill in a request.
+ */
+ miscq_msg_inbuf_add(vb, vb->miscq_msg_rup);
+ }
+
+ kfree(names);
+ kfree(callbacks);
+ kfree(vqs);
return 0;
+
+err_find:
+ kfree(names);
+err_names:
+ kfree(callbacks);
+err_callback:
+ kfree(vqs);
+err_vq:
+ return err;
}
#ifdef CONFIG_BALLOON_COMPACTION
@@ -723,7 +921,7 @@ static int init_vqs(struct virtio_balloon *vb)
static void tell_host_one_page(struct virtio_balloon *vb,
struct virtqueue *vq, struct page *page)
{
- add_one_chunk(vb, vq, page_to_pfn(page),
+ add_one_chunk(vb, vq, PAGE_CHUNK_TYPE_BALLOON, page_to_pfn(page),
VIRTIO_BALLOON_PAGES_PER_PAGE);
}
@@ -857,6 +1055,22 @@ static int balloon_page_chunk_init(struct virtio_balloon *vb)
return -ENOMEM;
}
+static int miscq_init(struct virtio_balloon *vb)
+{
+ vb->miscq_msg_rup = kmalloc(sizeof(struct virtio_balloon_miscq_msg) +
+ sizeof(struct virtio_balloon_page_chunk_entry) *
+ VIRTIO_BALLOON_MAX_PAGE_CHUNKS, GFP_KERNEL);
+ if (!vb->miscq_msg_rup) {
+ __virtio_clear_bit(vb->vdev, VIRTIO_BALLOON_F_MISC_VQ);
+ dev_warn(&vb->vdev->dev, "%s: failed\n", __func__);
+ return -ENOMEM;
+ }
+
+ INIT_WORK(&vb->miscq_handle_work, miscq_handle_func);
+
+ return 0;
+}
+
static int virtballoon_validate(struct virtio_device *vdev)
{
struct virtio_balloon *vb = NULL;
@@ -874,7 +1088,16 @@ static int virtballoon_validate(struct virtio_device *vdev)
goto err_page_chunk;
}
+ if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_MISC_VQ)) {
+ err = miscq_init(vb);
+ if (err < 0)
+ goto err_miscq_rup;
+ }
+
return 0;
+err_miscq_rup:
+ free_page_bmap(vb);
+ kfree(vb->balloon_page_chunk);
err_page_chunk:
kfree(vb);
err_vb:
@@ -971,6 +1194,7 @@ static void virtballoon_remove(struct virtio_device *vdev)
spin_unlock_irq(&vb->stop_update_lock);
cancel_work_sync(&vb->update_balloon_size_work);
cancel_work_sync(&vb->update_balloon_stats_work);
+ cancel_work_sync(&vb->miscq_handle_work);
remove_common(vb);
free_page_bmap(vb);
@@ -1020,6 +1244,7 @@ static unsigned int features[] = {
VIRTIO_BALLOON_F_STATS_VQ,
VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
VIRTIO_BALLOON_F_PAGE_CHUNKS,
+ VIRTIO_BALLOON_F_MISC_VQ,
};
static struct virtio_driver virtio_balloon_driver = {
@@ -35,6 +35,7 @@
#define VIRTIO_BALLOON_F_STATS_VQ 1 /* Memory Stats virtqueue */
#define VIRTIO_BALLOON_F_DEFLATE_ON_OOM 2 /* Deflate balloon on OOM */
#define VIRTIO_BALLOON_F_PAGE_CHUNKS 3 /* Inflate/Deflate pages in chunks */
+#define VIRTIO_BALLOON_F_MISC_VQ 4 /* Virtqueue for misc. requests */
/* Size of a PFN in the balloon interface. */
#define VIRTIO_BALLOON_PFN_SHIFT 12
@@ -96,4 +97,15 @@ struct virtio_balloon_page_chunk {
struct virtio_balloon_page_chunk_entry entry[];
};
+struct virtio_balloon_miscq_msg {
+#define VIRTIO_BALLOON_MISCQ_CMD_REPORT_UNUSED_PAGES 0
+ __le32 cmd;
+/* Flag to indicate the completion of handling a command */
+#define VIRTIO_BALLOON_MISCQ_F_COMPLETION 1
+ __le32 flags;
+ union {
+ struct virtio_balloon_page_chunk chunk;
+ } payload;
+};
+
#endif /* _LINUX_VIRTIO_BALLOON_H */