@@ -67,9 +67,16 @@
#define LAST_ADD_TIME_INVALID(vq)
#endif
+struct vring_split_desc_indir {
+ dma_addr_t addr; /* Descriptor Array DMA addr. */
+ u32 len; /* Descriptor Array length. */
+ u32 num;
+ struct vring_desc desc[];
+};
+
struct vring_desc_state_split {
void *data; /* Data for callback. */
- struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
+ struct vring_split_desc_indir *indir_desc; /* Indirect descriptor, if any. */
};
struct vring_packed_desc_indir {
@@ -478,12 +485,16 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
return extra[i].next;
}
-static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
- unsigned int total_sg,
- gfp_t gfp)
+static struct vring_split_desc_indir *alloc_indirect_split(struct virtqueue *_vq,
+ unsigned int total_sg,
+ gfp_t gfp)
{
+ struct vring_split_desc_indir *in_desc;
struct vring_desc *desc;
unsigned int i;
+ u32 size;
+
+ size = struct_size(in_desc, desc, total_sg);
/*
* We require lowmem mappings for the descriptors because
@@ -492,13 +503,16 @@ static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
*/
gfp &= ~__GFP_HIGHMEM;
- desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
- if (!desc)
+ in_desc = kmalloc(size, gfp);
+ if (!in_desc)
return NULL;
+ desc = in_desc->desc;
+
for (i = 0; i < total_sg; i++)
desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
- return desc;
+
+ return in_desc;
}
static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
@@ -540,6 +554,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
gfp_t gfp)
{
struct vring_virtqueue *vq = to_vvq(_vq);
+ struct vring_split_desc_indir *in_desc;
struct scatterlist *sg;
struct vring_desc *desc;
unsigned int i, n, avail, descs_used, prev, err_idx;
@@ -562,9 +577,13 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
head = vq->free_head;
- if (virtqueue_use_indirect(vq, total_sg))
- desc = alloc_indirect_split(_vq, total_sg, gfp);
- else {
+ if (virtqueue_use_indirect(vq, total_sg)) {
+ in_desc = alloc_indirect_split(_vq, total_sg, gfp);
+ if (!in_desc)
+ desc = NULL;
+ else
+ desc = in_desc->desc;
+ } else {
desc = NULL;
WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
}
@@ -637,10 +656,10 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
~VRING_DESC_F_NEXT;
if (indirect) {
+ u32 size = total_sg * sizeof(struct vring_desc);
+
/* Now that the indirect table is filled in, map it. */
- dma_addr_t addr = vring_map_single(
- vq, desc, total_sg * sizeof(struct vring_desc),
- DMA_TO_DEVICE);
+ dma_addr_t addr = vring_map_single(vq, desc, size, DMA_TO_DEVICE);
if (vring_mapping_error(vq, addr)) {
if (!vring_need_unmap_buffer(vq))
goto free_indirect;
@@ -648,11 +667,17 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
goto unmap_release;
}
- virtqueue_add_desc_split(_vq, vq->split.vring.desc,
- head, addr,
- total_sg * sizeof(struct vring_desc),
- VRING_DESC_F_INDIRECT,
- false);
+ desc = &vq->split.vring.desc[head];
+
+ desc->flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);
+ desc->addr = cpu_to_virtio64(_vq->vdev, addr);
+ desc->len = cpu_to_virtio32(_vq->vdev, size);
+
+ vq->split.desc_extra[head].flags = VRING_DESC_F_INDIRECT;
+
+ in_desc->addr = addr;
+ in_desc->len = size;
+ in_desc->num = total_sg;
}
/* We're using some buffers from the free list. */
@@ -667,7 +692,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
/* Store token and indirect buffer state. */
vq->split.desc_state[head].data = data;
if (indirect)
- vq->split.desc_state[head].indir_desc = desc;
+ vq->split.desc_state[head].indir_desc = in_desc;
else
vq->split.desc_state[head].indir_desc = ctx;
@@ -717,7 +742,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
free_indirect:
if (indirect)
- kfree(desc);
+ kfree(in_desc);
END_USE(vq);
return -ENOMEM;
@@ -782,34 +807,27 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
if (ctx)
*ctx = vq->split.desc_state[head].indir_desc;
} else {
- struct vring_desc *indir_desc =
- vq->split.desc_state[head].indir_desc;
- u32 len;
+ struct vring_split_desc_indir *in_desc;
- if (vq->use_dma_api) {
- struct vring_desc_extra *extra = vq->split.desc_extra;
+ in_desc = vq->split.desc_state[head].indir_desc;
+ if (vq->use_dma_api) {
dma_unmap_single(vring_dma_dev(vq),
- extra[i].addr,
- extra[i].len,
+ in_desc->addr, in_desc->len,
(flags & VRING_DESC_F_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE);
}
- len = vq->split.desc_extra[head].len;
-
BUG_ON(!(vq->split.desc_extra[head].flags &
VRING_DESC_F_INDIRECT));
- BUG_ON(len == 0 || len % sizeof(struct vring_desc));
if (vring_need_unmap_buffer(vq)) {
- for (j = 0; j < len / sizeof(struct vring_desc); j++)
- vring_unmap_one_split_indirect(vq, &indir_desc[j]);
+ for (j = 0; j < in_desc->num; j++)
+ vring_unmap_one_split_indirect(vq, &in_desc->desc[j]);
}
- kfree(indir_desc);
+ kfree(in_desc);
vq->split.desc_state[head].indir_desc = NULL;
-
}
vq->split.desc_extra[i].next = vq->free_head;
This commit structure the indirect desc table. Then we can get the desc num directly when doing unmap. And save the dma info to the struct, then the indirect will not use the dma fields of the desc_extra. The subsequent commits will make the dma fields are optional. But for the indirect case, we must record the dma info. Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> --- drivers/virtio/virtio_ring.c | 86 ++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 34 deletions(-)