@@ -525,6 +525,9 @@ void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
struct bio_vec bv;
struct bvec_iter iter;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return;
+
__bio_for_each_segment(bv, bio, iter, start) {
char *data = bvec_kmap_irq(&bv, &flags);
memset(data, 0, bv.bv_len);
@@ -707,6 +710,8 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,
*/
if (unlikely(bio_flagged(bio, BIO_CLONED)))
return 0;
+ if (unlikely(bio_is_dma_direct(bio)))
+ return 0;
if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
return 0;
@@ -783,6 +788,8 @@ bool __bio_try_merge_page(struct bio *bio, struct page *page,
{
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
return false;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return false;
if (bio->bi_vcnt > 0) {
struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
@@ -814,6 +821,7 @@ void __bio_add_page(struct bio *bio, struct page *page,
WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
WARN_ON_ONCE(bio_full(bio));
+ WARN_ON_ONCE(bio_is_dma_direct(bio));
bv->bv_page = page;
bv->bv_offset = off;
@@ -851,6 +859,8 @@ static void bio_get_pages(struct bio *bio)
struct bvec_iter_all iter_all;
struct bio_vec *bvec;
+ WARN_ON_ONCE(bio_is_dma_direct(bio));
+
bio_for_each_segment_all(bvec, bio, iter_all)
get_page(bvec->bv_page);
}
@@ -956,6 +966,8 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
if (WARN_ON_ONCE(bio->bi_vcnt))
return -EINVAL;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return -EINVAL;
do {
if (is_bvec)
@@ -1029,6 +1041,9 @@ void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
void *src_p, *dst_p;
unsigned bytes;
+ if (WARN_ON_ONCE(bio_is_dma_direct(src) || bio_is_dma_direct(dst)))
+ return;
+
while (src_iter->bi_size && dst_iter->bi_size) {
src_bv = bio_iter_iovec(src, *src_iter);
dst_bv = bio_iter_iovec(dst, *dst_iter);
@@ -1143,6 +1158,9 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return -EINVAL;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
ssize_t ret;
@@ -1174,6 +1192,9 @@ static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return -EINVAL;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
ssize_t ret;
@@ -1197,6 +1218,9 @@ void bio_free_pages(struct bio *bio)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return;
+
bio_for_each_segment_all(bvec, bio, iter_all)
__free_page(bvec->bv_page);
}
@@ -1653,6 +1677,9 @@ void bio_set_pages_dirty(struct bio *bio)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (unlikely(bio_is_dma_direct(bio)))
+ return;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
if (!PageCompound(bvec->bv_page))
set_page_dirty_lock(bvec->bv_page);
@@ -1704,6 +1731,9 @@ void bio_check_pages_dirty(struct bio *bio)
unsigned long flags;
struct bvec_iter_all iter_all;
+ if (unlikely(bio_is_dma_direct(bio)))
+ return;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
goto defer;
@@ -1777,6 +1807,9 @@ void bio_flush_dcache_pages(struct bio *bi)
struct bio_vec bvec;
struct bvec_iter iter;
+ if (unlikely(bio_is_dma_direct(bi)))
+ return;
+
bio_for_each_segment(bvec, bi, iter)
flush_dcache_page(bvec.bv_page);
}
@@ -1467,6 +1467,9 @@ void rq_flush_dcache_pages(struct request *rq)
struct req_iterator iter;
struct bio_vec bvec;
+ if (unlikely(blk_rq_is_dma_direct(rq)))
+ return;
+
rq_for_each_segment(bvec, rq, iter)
flush_dcache_page(bvec.bv_page);
}
This is a result of an audit of users of 'bi_io_vec'. A number of warnings and blocking conditions are added to ensure dma-direct bios are not incorrectly accessing the 'bi_io_vec' when they should access the 'bi_dma_vec'. These are largely just protecting against mis-uses in future development so depending on taste and public opinion some or all of these checks may not be necessary. A few other issues with dma-direct bios will be tackled in subsequent patches. Signed-off-by: Logan Gunthorpe <logang@deltatee.com> --- block/bio.c | 33 +++++++++++++++++++++++++++++++++ block/blk-core.c | 3 +++ 2 files changed, 36 insertions(+)