@@ -43,6 +43,8 @@ void blk_flush_integrity(void)
* @bio: bio to attach integrity metadata to
* @gfp_mask: Memory allocation mask
* @nr_vecs: Number of integrity metadata scatter-gather elements
+ * @profile: Optional integrity functions overrides the ones registered
+ * on the block device
*
* Description: This function prepares a bio for attaching integrity
* metadata. nr_vecs specifies the maximum number of pages containing
@@ -50,7 +52,8 @@ void blk_flush_integrity(void)
*/
struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
gfp_t gfp_mask,
- unsigned int nr_vecs)
+ unsigned int nr_vecs,
+ const struct blk_integrity_profile *profile)
{
struct bio_integrity_payload *bip;
struct bio_set *bs = bio->bi_pool;
@@ -85,6 +88,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
}
bip->bip_bio = bio;
+ bip->bip_profile = profile;
bio->bi_integrity = bip;
bio->bi_opf |= REQ_INTEGRITY;
@@ -224,16 +228,20 @@ static blk_status_t bio_integrity_process(struct bio *bio,
/**
* bio_integrity_prep - Prepare bio for integrity I/O
* @bio: bio to prepare
+ * @profile: Optional integrity functions specific to this bio
*
* Description: Checks if the bio already has an integrity payload attached.
* If it does, the payload has been generated by another kernel subsystem,
* and we just pass it through. Otherwise allocates integrity payload.
* The bio must have data direction, target device and start sector set priot
- * to calling. In the WRITE case, integrity metadata will be generated using
- * the block device's integrity function. In the READ case, the buffer
- * will be prepared for DMA and a suitable end_io handler set up.
+ * to calling. The optional integrity functions take priority over the ones
+ * from the block device if provided. In the WRITE case, integrity metadata
+ * will be generated using the appropriate integrity function.
+ * In the READ case, the buffer will be prepared for DMA and a suitable
+ * end_io handler set up.
*/
-bool bio_integrity_prep(struct bio *bio)
+bool bio_integrity_prep(struct bio *bio,
+ const struct blk_integrity_profile *profile)
{
struct bio_integrity_payload *bip;
struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
@@ -283,7 +291,7 @@ bool bio_integrity_prep(struct bio *bio)
nr_pages = end - start;
/* Allocate bio integrity payload and integrity vectors */
- bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);
+ bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages, profile);
if (IS_ERR(bip)) {
printk(KERN_ERR "could not allocate data integrity bioset\n");
kfree(buf);
@@ -326,8 +334,11 @@ bool bio_integrity_prep(struct bio *bio)
/* Auto-generate integrity metadata if this is a write */
if (bio_data_dir(bio) == WRITE) {
- bio_integrity_process(bio, &bio->bi_iter,
- bi->profile->generate_fn);
+ integrity_processing_fn *generate_fn =
+ (bip->bip_profile && bip->bip_profile->generate_fn) ?
+ bip->bip_profile->generate_fn :
+ bi->profile->generate_fn;
+ bio_integrity_process(bio, &bio->bi_iter, generate_fn);
}
return true;
@@ -354,6 +365,10 @@ static void bio_integrity_verify_fn(struct work_struct *work)
struct bio *bio = bip->bip_bio;
struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
struct bvec_iter iter = bio->bi_iter;
+ integrity_processing_fn *verify_fn =
+ (bip->bip_profile && bip->bip_profile->verify_fn) ?
+ bip->bip_profile->verify_fn :
+ bi->profile->verify_fn;
/*
* At the moment verify is called bio's iterator was advanced
@@ -361,8 +376,7 @@ static void bio_integrity_verify_fn(struct work_struct *work)
* it's original position.
*/
if (bio_rewind_iter(bio, &iter, iter.bi_done)) {
- bio->bi_status = bio_integrity_process(bio, &iter,
- bi->profile->verify_fn);
+ bio->bi_status = bio_integrity_process(bio, &iter, verify_fn);
} else {
bio->bi_status = BLK_STS_IOERR;
}
@@ -449,7 +463,8 @@ int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
BUG_ON(bip_src == NULL);
- bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt);
+ bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt,
+ bip_src->bip_profile);
if (IS_ERR(bip))
return PTR_ERR(bip);
@@ -1993,7 +1993,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
blk_queue_split(q, &bio);
- if (!bio_integrity_prep(bio))
+ if (!bio_integrity_prep(bio, NULL))
return BLK_QC_T_NONE;
if (op_is_flush(bio->bi_opf)) {
@@ -1762,7 +1762,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
blk_queue_split(q, &bio);
- if (!bio_integrity_prep(bio))
+ if (!bio_integrity_prep(bio, NULL))
return BLK_QC_T_NONE;
if (!is_flush_fua && !blk_queue_nomerges(q) &&
@@ -931,7 +931,7 @@ static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
if (!bio_sectors(bio) || !io->cc->on_disk_tag_size)
return 0;
- bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
+ bip = bio_integrity_alloc(bio, GFP_NOIO, 1, NULL);
if (IS_ERR(bip))
return PTR_ERR(bip);
@@ -180,7 +180,7 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio)
int err = 0, rw;
bool do_acct;
- if (!bio_integrity_prep(bio))
+ if (!bio_integrity_prep(bio, NULL))
return BLK_QC_T_NONE;
bip = bio_integrity(bio);
@@ -1448,7 +1448,7 @@ static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio)
int err = 0;
bool do_acct;
- if (!bio_integrity_prep(bio))
+ if (!bio_integrity_prep(bio, NULL))
return BLK_QC_T_NONE;
do_acct = nd_iostat_start(bio, &start);
@@ -736,7 +736,7 @@ static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
if (write && copy_from_user(buf, ubuf, len))
goto out_free_meta;
- bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
+ bip = bio_integrity_alloc(bio, GFP_KERNEL, 1, NULL);
if (IS_ERR(bip)) {
ret = PTR_ERR(bip);
goto out_free_meta;
@@ -650,7 +650,7 @@ iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio)
return -ENODEV;
}
- bip = bio_integrity_alloc(bio, GFP_NOIO, cmd->t_prot_nents);
+ bip = bio_integrity_alloc(bio, GFP_NOIO, cmd->t_prot_nents, NULL);
if (IS_ERR(bip)) {
pr_err("Unable to allocate bio_integrity_payload\n");
return PTR_ERR(bip);
@@ -355,6 +355,9 @@ struct bio_integrity_payload {
struct work_struct bip_work; /* I/O completion */
+ /* integrity profile specific to the parent bio */
+ const struct blk_integrity_profile *bip_profile;
+
struct bio_vec *bip_vec;
struct bio_vec bip_inline_vecs[0];/* embedded bvec array */
};
@@ -788,9 +791,12 @@ static inline bool bioset_initialized(struct bio_set *bs)
for_each_bio(_bio) \
bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
-extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
+extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t,
+ unsigned int,
+ const struct blk_integrity_profile *);
extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
-extern bool bio_integrity_prep(struct bio *);
+extern bool bio_integrity_prep(struct bio *,
+ const struct blk_integrity_profile *);
extern void bio_integrity_advance(struct bio *, unsigned int);
extern void bio_integrity_trim(struct bio *);
extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
@@ -815,7 +821,8 @@ static inline void bioset_integrity_free (struct bio_set *bs)
return;
}
-static inline bool bio_integrity_prep(struct bio *bio)
+static inline bool bio_integrity_prep(struct bio *bio,
+ const struct blk_integrity_profile *profile)
{
return true;
}
@@ -848,7 +855,8 @@ static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
}
static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,
- unsigned int nr)
+ unsigned int nr,
+ const struct blk_integrity_profile *profile)
{
return ERR_PTR(-EINVAL);
}