@@ -28,6 +28,8 @@
#define bio_iter_iovec(bio, iter) \
bvec_iter_bvec((bio)->bi_io_vec, (iter))
+#define bio_iter_dma_vec(bio, iter) \
+ bvec_iter_dvec((bio)->bi_dma_vec, (iter))
#define bio_iter_page(bio, iter) \
bvec_iter_page((bio)->bi_io_vec, (iter))
@@ -39,6 +41,7 @@
#define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter)
#define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter)
#define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter)
+#define bio_dma_vec(bio) bio_iter_dma_vec((bio), (bio)->bi_iter)
#define bio_multiple_segments(bio) \
((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)
@@ -155,6 +158,15 @@ static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
#define bio_for_each_bvec(bvl, bio, iter) \
__bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
+#define __bio_for_each_dvec(dvl, bio, iter, start) \
+ for (iter = (start); \
+ (iter).bi_size && \
+ ((dvl = bvec_iter_dvec((bio)->bi_dma_vec, (iter))), 1); \
+ dvec_iter_advance((bio)->bi_dma_vec, &(iter), (dvl).dv_len))
+
+#define bio_for_each_dvec(dvl, bio, iter) \
+ __bio_for_each_dvec(dvl, bio, iter, (bio)->bi_iter)
+
#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
static inline unsigned bio_segments(struct bio *bio)
@@ -191,7 +191,10 @@ struct bio {
atomic_t __bi_cnt; /* pin count */
- struct bio_vec *bi_io_vec; /* the actual vec list */
+ union {
+ struct bio_vec *bi_io_vec; /* the actual vec list */
+ struct dma_vec *bi_dma_vec; /* for dma direct bios*/
+ };
struct bio_set *bi_pool;
@@ -21,6 +21,11 @@ struct bio_vec {
unsigned int bv_offset;
};
+struct dma_vec {
+ dma_addr_t dv_addr;
+ unsigned int dv_len;
+};
+
struct bvec_iter {
sector_t bi_sector; /* device address in 512 byte
sectors */
@@ -84,6 +89,18 @@ struct bvec_iter_all {
.bv_offset = bvec_iter_offset((bvec), (iter)), \
})
+#define bvec_iter_dvec_addr(dvec, iter) \
+ (__bvec_iter_bvec((dvec), (iter))->dv_addr + (iter).bi_bvec_done)
+#define bvec_iter_dvec_len(dvec, iter) \
+ min((iter).bi_size, \
+ __bvec_iter_bvec((dvec), (iter))->dv_len - (iter).bi_bvec_done)
+
+#define bvec_iter_dvec(dvec, iter) \
+((struct dma_vec) { \
+ .dv_addr = bvec_iter_dvec_addr((dvec), (iter)), \
+ .dv_len = bvec_iter_dvec_len((dvec), (iter)), \
+})
+
static inline bool bvec_iter_advance(const struct bio_vec *bv,
struct bvec_iter *iter, unsigned bytes)
{
@@ -110,6 +127,32 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
return true;
}
+static inline bool dvec_iter_advance(const struct dma_vec *dv,
+ struct bvec_iter *iter, unsigned bytes)
+{
+ if (WARN_ONCE(bytes > iter->bi_size,
+ "Attempted to advance past end of dvec iter\n")) {
+ iter->bi_size = 0;
+ return false;
+ }
+
+ while (bytes) {
+ const struct dma_vec *cur = dv + iter->bi_idx;
+ unsigned len = min3(bytes, iter->bi_size,
+ cur->dv_len - iter->bi_bvec_done);
+
+ bytes -= len;
+ iter->bi_size -= len;
+ iter->bi_bvec_done += len;
+
+ if (iter->bi_bvec_done == cur->dv_len) {
+ iter->bi_bvec_done = 0;
+ iter->bi_idx++;
+ }
+ }
+ return true;
+}
+
#define for_each_bvec(bvl, bio_vec, iter, start) \
for (iter = (start); \
(iter).bi_size && \
The dma_vec structure is similar to the bio_vec structure except it only stores DMA addresses instead of the struct page address. struct bios will be able to make use of dma_vecs with a union and, therefore, we need to ensure that struct dma_vec is no larger than struct bvec, as they will share the allocated memory. dma_vecs can make the same use of the bvec_iter structure to iterate through the vectors. This will be used for passing DMA addresses directly through the block layer. I expect something like struct dma_vec will also be used in Christoph's work to improve the dma_mapping layer and remove sgls. At some point, these would use the same structure. Signed-off-by: Logan Gunthorpe <logang@deltatee.com> --- include/linux/bio.h | 12 +++++++++++ include/linux/blk_types.h | 5 ++++- include/linux/bvec.h | 43 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-)