From: Alasdair G Kergon Further experimental cleanup of clone_bio code. Add core num_write_bios support. - split this into 3: renames, code moves, num_write_bios --- drivers/md/dm.c | 236 ++++++++++++++++++++++++------------------ include/linux/device-mapper.h | 15 ++ 2 files changed, 153 insertions(+), 98 deletions(-) Index: linux/drivers/md/dm.c =================================================================== --- linux.orig/drivers/md/dm.c +++ linux/drivers/md/dm.c @@ -1063,9 +1063,9 @@ static void clone_bio_integrity(struct b /* * Creates a little bio that just does part of a bvec. */ -static void split_bvec(struct dm_target_io *tio, struct bio *bio, - sector_t sector, unsigned short idx, unsigned int offset, - unsigned int len) +static void clone_split_bio(struct dm_target_io *tio, struct bio *bio, + sector_t sector, unsigned short idx, + unsigned offset, unsigned len) { struct bio *clone = &tio->clone; struct bio_vec *bv = bio->bi_io_vec + idx; @@ -1121,10 +1121,40 @@ static struct dm_target_io *alloc_tio(st return tio; } -static void __issue_target_request(struct clone_info *ci, struct dm_target *ti, - unsigned target_bio_nr, sector_t len) +static void __clone_and_map_data_bio(struct clone_info *ci, + struct dm_target *ti, + sector_t sector, int nr_iovecs, + unsigned short idx, unsigned bv_count, + unsigned len, + unsigned split_bvec) { - struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs, target_bio_nr); + struct bio *bio = ci->bio; + struct dm_target_io *tio; + unsigned target_bio_nr; + unsigned num_target_bios = 1; + + /* + * Does the target want to receive duplicate copies of the bio? + */ + if (bio_data_dir(bio) == WRITE && ti->num_write_bios) + num_target_bios = ti->num_write_bios(ti, bio); + + for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) { + tio = alloc_tio(ci, ti, nr_iovecs, target_bio_nr); + if (split_bvec) + clone_split_bio(tio, bio, sector, idx, bv_count, len); + else + clone_bio(tio, bio, sector, idx, bv_count, len); + __map_bio(tio); + } +} + +static void __clone_and_map_simple_bio(struct clone_info *ci, + struct dm_target *ti, + unsigned target_bio_nr, sector_t len) +{ + struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs, + target_bio_nr); struct bio *clone = &tio->clone; /* @@ -1139,42 +1169,27 @@ static void __issue_target_request(struc __map_bio(tio); } -static void __issue_target_bios(struct clone_info *ci, struct dm_target *ti, - unsigned num_bios, sector_t len) +static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, + unsigned num_bios, sector_t len) { unsigned target_bio_nr; for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++) - __issue_target_request(ci, ti, target_bio_nr, len); + __clone_and_map_simple_bio(ci, ti, target_bio_nr, len); } -static int __clone_and_map_empty_flush(struct clone_info *ci) +static int __send_empty_flush(struct clone_info *ci) { unsigned target_nr = 0; struct dm_target *ti; BUG_ON(bio_has_data(ci->bio)); while ((ti = dm_table_get_target(ci->map, target_nr++))) - __issue_target_bios(ci, ti, ti->num_flush_bios, 0); + __send_duplicate_bios(ci, ti, ti->num_flush_bios, 0); return 0; } -/* - * Perform all io with a single clone. - */ -static void __clone_and_map_simple(struct clone_info *ci, struct dm_target *ti) -{ - struct bio *bio = ci->bio; - struct dm_target_io *tio; - - tio = alloc_tio(ci, ti, bio->bi_max_vecs, 0); - clone_bio(tio, bio, ci->sector, ci->idx, bio->bi_vcnt - ci->idx, - ci->sector_count); - __map_bio(tio); - ci->sector_count = 0; -} - typedef unsigned (*get_num_bios_fn)(struct dm_target *ti); static unsigned get_num_discard_bios(struct dm_target *ti) @@ -1194,9 +1209,9 @@ static bool is_split_required_for_discar return ti->split_discard_bios; } -static int __clone_and_map_changing_extent_only(struct clone_info *ci, - get_num_bios_fn get_num_bios, - is_split_required_fn is_split_required) +static int __send_changing_extent_only(struct clone_info *ci, + get_num_bios_fn get_num_bios, + is_split_required_fn is_split_required) { struct dm_target *ti; sector_t len; @@ -1222,7 +1237,7 @@ static int __clone_and_map_changing_exte else len = min(ci->sector_count, max_io_len(ci->sector, ti)); - __issue_target_bios(ci, ti, num_bios, len); + __send_duplicate_bios(ci, ti, num_bios, len); ci->sector += len; } while (ci->sector_count -= len); @@ -1230,28 +1245,86 @@ static int __clone_and_map_changing_exte return 0; } -static int __clone_and_map_discard(struct clone_info *ci) +static int __send_discard(struct clone_info *ci) { - return __clone_and_map_changing_extent_only(ci, get_num_discard_bios, - is_split_required_for_discard); + return __send_changing_extent_only(ci, get_num_discard_bios, + is_split_required_for_discard); } -static int __clone_and_map_write_same(struct clone_info *ci) +static int __send_write_same(struct clone_info *ci) +{ + return __send_changing_extent_only(ci, get_num_write_same_bios, NULL); +} + +/* + * Find maximum number of sectors / bvecs we can process with a single bio. + */ +static sector_t __len_within_target(struct clone_info *ci, sector_t max, + int *idx) { - return __clone_and_map_changing_extent_only(ci, get_num_write_same_bios, NULL); + struct bio *bio = ci->bio; + sector_t bv_len, total_len = 0; + + for (*idx = ci->idx; max && (*idx < bio->bi_vcnt); (*idx)++) { + bv_len = to_sector(bio->bi_io_vec[*idx].bv_len); + + if (bv_len > max) + break; + + max -= bv_len; + total_len += bv_len; + } + + return total_len; } +static int __split_bvec_across_targets(struct clone_info *ci, + struct dm_target *ti, sector_t max) +{ + struct bio *bio = ci->bio; + struct bio_vec *bv = bio->bi_io_vec + ci->idx; + sector_t remaining = to_sector(bv->bv_len); + unsigned offset = 0; + sector_t len; + + do { + if (offset) { + ti = dm_table_find_target(ci->map, ci->sector); + if (!dm_target_is_valid(ti)) + return -EIO; + + max = max_io_len(ci->sector, ti); + } + + len = min(remaining, max); + + __clone_and_map_data_bio(ci, ti, ci->sector, 1, ci->idx, + bv->bv_offset + offset, len, 1); + + ci->sector += len; + ci->sector_count -= len; + offset += to_bytes(len); + } while (remaining -= len); + + ci->idx++; + + return 0; +} + +/* + * Select the correct strategy to for processing a bio. + */ static int __clone_and_map(struct clone_info *ci) { struct bio *bio = ci->bio; struct dm_target *ti; - sector_t len = 0, max; - struct dm_target_io *tio; + sector_t len, max; + int idx; if (unlikely(bio->bi_rw & REQ_DISCARD)) - return __clone_and_map_discard(ci); + return __send_discard(ci); else if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) - return __clone_and_map_write_same(ci); + return __send_write_same(ci); ti = dm_table_find_target(ci->map, ci->sector); if (!dm_target_is_valid(ti)) @@ -1259,78 +1332,45 @@ static int __clone_and_map(struct clone_ max = max_io_len(ci->sector, ti); + /* + * Optimise for the simple case where we can do all of + * the remaining io with a single clone. + */ if (ci->sector_count <= max) { - /* - * Optimise for the simple case where we can do all of - * the remaining io with a single clone. - */ - __clone_and_map_simple(ci, ti); - - } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) { - /* - * There are some bvecs that don't span targets. - * Do as many of these as possible. - */ - int i; - sector_t remaining = max; - sector_t bv_len; + __clone_and_map_data_bio(ci, ti, ci->sector, bio->bi_max_vecs, + ci->idx, bio->bi_vcnt - ci->idx, + ci->sector_count, 0); - for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) { - bv_len = to_sector(bio->bi_io_vec[i].bv_len); + ci->sector_count = 0; - if (bv_len > remaining) - break; + return 0; + } - remaining -= bv_len; - len += bv_len; - } + /* + * There are some bvecs that don't span targets. + * Do as many of these as possible. + */ + if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) { + len = __len_within_target(ci, max, &idx); - tio = alloc_tio(ci, ti, bio->bi_max_vecs, 0); - clone_bio(tio, bio, ci->sector, ci->idx, i - ci->idx, len); - __map_bio(tio); + __clone_and_map_data_bio(ci, ti, ci->sector, bio->bi_max_vecs, + ci->idx, idx - ci->idx, len, 0); ci->sector += len; ci->sector_count -= len; - ci->idx = i; - - } else { - /* - * Handle a bvec that must be split between two or more targets. - */ - struct bio_vec *bv = bio->bi_io_vec + ci->idx; - sector_t remaining = to_sector(bv->bv_len); - unsigned int offset = 0; + ci->idx = idx; - do { - if (offset) { - ti = dm_table_find_target(ci->map, ci->sector); - if (!dm_target_is_valid(ti)) - return -EIO; - - max = max_io_len(ci->sector, ti); - } - - len = min(remaining, max); - - tio = alloc_tio(ci, ti, 1, 0); - split_bvec(tio, bio, ci->sector, ci->idx, - bv->bv_offset + offset, len); - - __map_bio(tio); - - ci->sector += len; - ci->sector_count -= len; - offset += to_bytes(len); - } while (remaining -= len); - - ci->idx++; + return 0; } - return 0; + /* + * Handle a bvec that must be split between two or more targets. + */ + return __split_bvec_across_targets(ci, ti, max); } /* - * Split the bio into several clones and submit it to targets. + * Entry point to split the bio into several clones and submit it to targets. */ static void __split_and_process_bio(struct mapped_device *md, struct bio *bio) { @@ -1358,7 +1398,7 @@ static void __split_and_process_bio(stru if (bio->bi_rw & REQ_FLUSH) { ci.bio = &ci.md->flush_bio; ci.sector_count = 0; - error = __clone_and_map_empty_flush(&ci); + error = __send_empty_flush(&ci); /* dec_pending submits any data associated with flush */ } else { ci.bio = bio; Index: linux/include/linux/device-mapper.h =================================================================== --- linux.orig/include/linux/device-mapper.h +++ linux/include/linux/device-mapper.h @@ -175,6 +175,14 @@ struct target_type { #define DM_TARGET_IMMUTABLE 0x00000004 #define dm_target_is_immutable(type) ((type)->features & DM_TARGET_IMMUTABLE) +/* + * Some targets need to be sent the same WRITE bio severals times so + * that they can send copies of it to different devices. This function + * examines any supplied bio and returns the number of copies of it the + * target requires. + */ +typedef unsigned (*dm_num_write_bios_fn) (struct dm_target *ti, struct bio *bio); + struct dm_target { struct dm_table *table; struct target_type *type; @@ -214,6 +222,13 @@ struct dm_target { */ unsigned per_bio_data_size; + /* + * If defined, this function is called to find out how many + * duplicate bios should be sent to the target when writing + * data. + */ + dm_num_write_bios_fn num_write_bios; + /* target specific data */ void *private;