// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2024-2025 Christoph Hellwig. */ #include #include #include "internal.h" struct bio_set iomap_ioend_bioset; EXPORT_SYMBOL_GPL(iomap_ioend_bioset); struct iomap_ioend *iomap_init_ioend(struct inode *inode, struct bio *bio, loff_t file_offset, u16 ioend_flags) { struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); atomic_set(&ioend->io_remaining, 1); ioend->io_error = 0; ioend->io_parent = NULL; INIT_LIST_HEAD(&ioend->io_list); ioend->io_flags = ioend_flags; ioend->io_inode = inode; ioend->io_offset = file_offset; ioend->io_size = bio->bi_iter.bi_size; ioend->io_sector = bio->bi_iter.bi_sector; ioend->io_private = NULL; return ioend; } EXPORT_SYMBOL_GPL(iomap_init_ioend); static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error) { if (ioend->io_parent) { struct bio *bio = &ioend->io_bio; ioend = ioend->io_parent; bio_put(bio); } if (error) cmpxchg(&ioend->io_error, 0, error); if (!atomic_dec_and_test(&ioend->io_remaining)) return 0; if (ioend->io_flags & IOMAP_IOEND_DIRECT) return iomap_finish_ioend_direct(ioend); return iomap_finish_ioend_buffered(ioend); } /* * Ioend completion routine for merged bios. This can only be called from task * contexts as merged ioends can be of unbound length. Hence we have to break up * the writeback completions into manageable chunks to avoid long scheduler * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get * good batch processing throughput without creating adverse scheduler latency * conditions. */ void iomap_finish_ioends(struct iomap_ioend *ioend, int error) { struct list_head tmp; u32 completions; might_sleep(); list_replace_init(&ioend->io_list, &tmp); completions = iomap_finish_ioend(ioend, error); while (!list_empty(&tmp)) { if (completions > IOEND_BATCH_SIZE * 8) { cond_resched(); completions = 0; } ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); list_del_init(&ioend->io_list); completions += iomap_finish_ioend(ioend, error); } } EXPORT_SYMBOL_GPL(iomap_finish_ioends); /* * We can merge two adjacent ioends if they have the same set of work to do. */ static bool iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next) { if (ioend->io_bio.bi_status != next->io_bio.bi_status) return false; if (next->io_flags & IOMAP_IOEND_BOUNDARY) return false; if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) != (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) return false; if (ioend->io_offset + ioend->io_size != next->io_offset) return false; /* * Do not merge physically discontiguous ioends. The filesystem * completion functions will have to iterate the physical * discontiguities even if we merge the ioends at a logical level, so * we don't gain anything by merging physical discontiguities here. * * We cannot use bio->bi_iter.bi_sector here as it is modified during * submission so does not point to the start sector of the bio at * completion. */ if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) != next->io_sector) return false; return true; } void iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends) { struct iomap_ioend *next; INIT_LIST_HEAD(&ioend->io_list); while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, io_list))) { if (!iomap_ioend_can_merge(ioend, next)) break; list_move_tail(&next->io_list, &ioend->io_list); ioend->io_size += next->io_size; } } EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); static int iomap_ioend_compare(void *priv, const struct list_head *a, const struct list_head *b) { struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); if (ia->io_offset < ib->io_offset) return -1; if (ia->io_offset > ib->io_offset) return 1; return 0; } void iomap_sort_ioends(struct list_head *ioend_list) { list_sort(NULL, ioend_list, iomap_ioend_compare); } EXPORT_SYMBOL_GPL(iomap_sort_ioends); /* * Split up to the first @max_len bytes from @ioend if the ioend covers more * than @max_len bytes. * * If @is_append is set, the split will be based on the hardware limits for * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware * limits don't allow the entire @max_len length. * * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to * switch the operation after this call, but before submitting the bio. */ struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend, unsigned int max_len, bool is_append) { struct bio *bio = &ioend->io_bio; struct iomap_ioend *split_ioend; unsigned int nr_segs; int sector_offset; struct bio *split; if (is_append) { struct queue_limits *lim = bdev_limits(bio->bi_bdev); max_len = min(max_len, lim->max_zone_append_sectors << SECTOR_SHIFT); sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len); if (unlikely(sector_offset < 0)) return ERR_PTR(sector_offset); if (!sector_offset) return NULL; } else { if (bio->bi_iter.bi_size <= max_len) return NULL; sector_offset = max_len >> SECTOR_SHIFT; } /* ensure the split ioend is still block size aligned */ sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT, i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT; split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset); if (IS_ERR(split)) return ERR_CAST(split); split->bi_private = bio->bi_private; split->bi_end_io = bio->bi_end_io; split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset, ioend->io_flags); split_ioend->io_parent = ioend; atomic_inc(&ioend->io_remaining); ioend->io_offset += split_ioend->io_size; ioend->io_size -= split_ioend->io_size; split_ioend->io_sector = ioend->io_sector; if (!is_append) ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT); return split_ioend; } EXPORT_SYMBOL_GPL(iomap_split_ioend); static int __init iomap_ioend_init(void) { return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), offsetof(struct iomap_ioend, io_bio), BIOSET_NEED_BVECS); } fs_initcall(iomap_ioend_init);