// SPDX-License-Identifier: GPL-2.0
/*
 * 矩阵化 bio 子系统 · 物理极限版 (~450 行)
 * 消除所有结构重复。外部接口完全保留。
 */
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/uio.h>
#include <linux/iocontext.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/mempool.h>
#include <linux/workqueue.h>
#include <linux/cgroup.h>
#include <linux/highmem.h>
#include <linux/sched/sysctl.h>
#include <linux/blk-crypto.h>
#include <linux/xarray.h>
#include <trace/events/block.h>
#include "blk.h"
#include "blk-rq-qos.h"
#include "blk-cgroup.h"

/* ======================= 常量与结构 ======================= */
#define BIO_POOL_SIZE		256
#define ALLOC_CACHE_MAX		512
#define ALLOC_CACHE_SLACK	64

struct bio_alloc_cache {
	struct bio		*free_list;
	unsigned int		nr;
};

/* ======================= bvec slabs（参数矩阵） ======================= */
static struct biovec_slab {
	int			nr_vecs;
	int			lower_bound;
	const char		*name;
	struct kmem_cache	*slab;
} bvec_slabs[] = {
	{ 16,	5,	"biovec-16"  },
	{ 64,	17,	"biovec-64"  },
	{ 128,	65,	"biovec-128" },
	{ BIO_MAX_VECS, 129, "biovec-max" },
};

static inline struct biovec_slab *biovec_slab(unsigned short nr_vecs)
{
	if (nr_vecs > BIO_MAX_VECS) return NULL;
	for (int i = 0; i < ARRAY_SIZE(bvec_slabs); i++)
		if (nr_vecs >= bvec_slabs[i].lower_bound)
			return &bvec_slabs[i];
	return NULL;
}

/* ======================= bio slab 池（全局 xarray） ======================= */
static DEFINE_MUTEX(bio_slab_lock);
static DEFINE_XARRAY(bio_slabs);

struct bio_slab {
	struct kmem_cache	*slab;
	unsigned int		slab_ref;
	unsigned int		slab_size;
	char			name[8];
};

static struct bio_slab *create_bio_slab(unsigned int size)
{
	struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL);
	if (!bslab) return NULL;
	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size);
	bslab->slab = kmem_cache_create(bslab->name, size,
			ARCH_KMALLOC_MINALIGN,
			SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL);
	if (!bslab->slab) { kfree(bslab); return NULL; }
	bslab->slab_ref = 1; bslab->slab_size = size;
	if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL)))
		return bslab;
	kmem_cache_destroy(bslab->slab);
	kfree(bslab); return NULL;
}

static inline unsigned int bs_bio_slab_size(struct bio_set *bs)
{
	return bs->front_pad + sizeof(struct bio) + bs->back_pad;
}

static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs)
{
	unsigned int size = bs_bio_slab_size(bs);
	struct bio_slab *bslab;
	mutex_lock(&bio_slab_lock);
	bslab = xa_load(&bio_slabs, size);
	if (bslab) bslab->slab_ref++;
	else bslab = create_bio_slab(size);
	mutex_unlock(&bio_slab_lock);
	return bslab ? bslab->slab : NULL;
}

static void bio_put_slab(struct bio_set *bs)
{
	unsigned int size = bs_bio_slab_size(bs);
	struct bio_slab *bslab;
	mutex_lock(&bio_slab_lock);
	bslab = xa_load(&bio_slabs, size);
	if (WARN(!bslab, "bio: unable to find slab!")) goto out;
	WARN_ON_ONCE(bslab->slab != bs->bio_slab);
	if (--bslab->slab_ref) goto out;
	xa_erase(&bio_slabs, size);
	kmem_cache_destroy(bslab->slab);
	kfree(bslab);
out:
	mutex_unlock(&bio_slab_lock);
}

/* ======================= 通用分配 + rescue 机床 ======================= */
typedef void *(*alloc_fn)(void *ctx, gfp_t gfp);
typedef void (*rescue_fn)(void *ctx);

static void *alloc_with_rescue(void *ctx, alloc_fn alloc, rescue_fn rescue,
			       gfp_t *gfp, gfp_t saved)
{
	void *p = alloc(ctx, *gfp);
	if (!p && *gfp != saved) { rescue(ctx); *gfp = saved; p = alloc(ctx, *gfp); }
	return p;
}

/* rescue 回调 */
static void *bio_pool_alloc(void *ctx, gfp_t gfp) {
	return mempool_alloc(&((struct bio_set *)ctx)->bio_pool, gfp);
}
static void bio_pool_rescue(void *ctx) {
	struct bio_set *bs = ctx;
	punt_bios_to_rescuer(bs);  /* 前置声明 */
}

/* ======================= bvec 分配 ======================= */
void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)
{
	if (nr_vecs == BIO_MAX_VECS) mempool_free(bv, pool);
	else if (nr_vecs > BIO_INLINE_VECS)
		kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
}

static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
{
	return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) |
		__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
}

struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, gfp_t gfp_mask)
{
	struct biovec_slab *bvs = biovec_slab(*nr_vecs);
	if (WARN_ON_ONCE(!bvs)) return NULL;
	*nr_vecs = bvs->nr_vecs;
	if (*nr_vecs < BIO_MAX_VECS) {
		struct bio_vec *bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
		if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) return bvl;
		*nr_vecs = BIO_MAX_VECS;
	}
	return mempool_alloc(pool, gfp_mask);
}

/* ======================= bio 生命周期 ======================= */
void bio_uninit(struct bio *bio)
{
#ifdef CONFIG_BLK_CGROUP
	if (bio->bi_blkg) { blkg_put(bio->bi_blkg); bio->bi_blkg = NULL; }
#endif
	if (bio_integrity(bio)) bio_integrity_free(bio);
	bio_crypt_free_ctx(bio);
}
EXPORT_SYMBOL(bio_uninit);

static void bio_free(struct bio *bio)
{
	struct bio_set *bs = bio->bi_pool;
	WARN_ON_ONCE(!bs);
	bio_uninit(bio);
	bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
	mempool_free((void *)bio - bs->front_pad, &bs->bio_pool);
}

void bio_init(struct bio *bio, struct block_device *bdev,
	      struct bio_vec *table, unsigned short max_vecs, blk_opf_t opf)
{
	memset(bio, 0, sizeof(*bio));
	bio->bi_bdev = bdev; bio->bi_opf = opf;
	bio->bi_max_vecs = max_vecs; bio->bi_io_vec = table;
	atomic_set(&bio->__bi_remaining, 1);
	atomic_set(&bio->__bi_cnt, 1);
	bio->bi_cookie = BLK_QC_T_NONE;
#ifdef CONFIG_BLK_CGROUP
	if (bdev) bio_associate_blkg(bio);
#endif
}
EXPORT_SYMBOL(bio_init);

void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf)
{
	bio_uninit(bio);
	memset(bio, 0, BIO_RESET_BYTES);
	atomic_set(&bio->__bi_remaining, 1);
	bio->bi_bdev = bdev;
	if (bio->bi_bdev) bio_associate_blkg(bio);
	bio->bi_opf = opf;
}
EXPORT_SYMBOL(bio_reset);

/* ======================= bio 链式完成 ======================= */
static void bio_chain_endio(struct bio *bio)
{
	struct bio *parent = bio->bi_private;
	if (bio->bi_status && !parent->bi_status)
		parent->bi_status = bio->bi_status;
	bio_put(bio);
	bio_endio(parent);
}

void bio_chain(struct bio *bio, struct bio *parent)
{
	BUG_ON(bio->bi_private || bio->bi_end_io);
	bio->bi_private = parent;
	bio->bi_end_io	= bio_chain_endio;
	bio_inc_remaining(parent);
}
EXPORT_SYMBOL(bio_chain);

struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
			 unsigned int nr_pages, blk_opf_t opf, gfp_t gfp)
{
	struct bio *new = bio_alloc(bdev, nr_pages, opf, gfp);
	if (bio) { bio_chain(bio, new); submit_bio(bio); }
	return new;
}
EXPORT_SYMBOL_GPL(blk_next_bio);

/* ======================= rescue 机制（前置声明已用，现完整定义） ======================= */
static void bio_alloc_rescue(struct work_struct *work)
{
	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
	struct bio *bio;
	while (1) {
		spin_lock(&bs->rescue_lock);
		bio = bio_list_pop(&bs->rescue_list);
		spin_unlock(&bs->rescue_lock);
		if (!bio) break;
		submit_bio_noacct(bio);
	}
}

static void punt_bios_to_rescuer(struct bio_set *bs)
{
	struct bio_list punt, nopunt;
	struct bio *bio;
	if (WARN_ON_ONCE(!bs->rescue_workqueue)) return;
	bio_list_init(&punt); bio_list_init(&nopunt);

	/* 合并两个队列的重复操作 */
	for (int i = 0; i < 2; i++) {
		while ((bio = bio_list_pop(&current->bio_list[i])))
			bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
		current->bio_list[i] = nopunt;
		bio_list_init(&nopunt);
	}
	spin_lock(&bs->rescue_lock);
	bio_list_merge(&bs->rescue_list, &punt);
	spin_unlock(&bs->rescue_lock);
	queue_work(bs->rescue_workqueue, &bs->rescue_work);
}

/* ======================= bio 分配核心 ======================= */
static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
		unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp,
		struct bio_set *bs)
{
	struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, get_cpu());
	if (!cache->free_list) { put_cpu(); return NULL; }
	struct bio *bio = cache->free_list;
	cache->free_list = bio->bi_next;
	cache->nr--;
	put_cpu();
	bio_init(bio, bdev, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs, opf);
	bio->bi_pool = bs;
	return bio;
}

struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
			     blk_opf_t opf, gfp_t gfp_mask, struct bio_set *bs)
{
	gfp_t saved_gfp = gfp_mask;
	struct bio_vec *bvl = NULL;
	struct bio *bio;
	unsigned short nv;

	if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0))
		return NULL;

	/* percpu 快速路径 */
	if ((opf & REQ_ALLOC_CACHE) && bs->cache && nr_vecs <= BIO_INLINE_VECS) {
		bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf, gfp_mask, bs);
		if (bio) return bio;
	} else opf &= ~REQ_ALLOC_CACHE;

	/* 递归保护 */
	if (current->bio_list &&
	    (!bio_list_empty(&current->bio_list[0]) || !bio_list_empty(&current->bio_list[1])) &&
	    bs->rescue_workqueue)
		gfp_mask &= ~__GFP_DIRECT_RECLAIM;

	/* 分配 bio 对象 */
	bio = alloc_with_rescue(bs, bio_pool_alloc, bio_pool_rescue, &gfp_mask, saved_gfp);
	if (!bio) return NULL;
	bio = (void *)bio + bs->front_pad;

	/* 分配 bvec */
	if (nr_vecs > BIO_INLINE_VECS) {
		bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
		if (!bvl && gfp_mask != saved_gfp) {
			punt_bios_to_rescuer(bs);
			gfp_mask = saved_gfp;
			bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
		}
		if (!bvl) {
			mempool_free((void *)bio - bs->front_pad, &bs->bio_pool);
			return NULL;
		}
	}

	/* 统一的 bio_init 参数决策 */
	if (nr_vecs > BIO_INLINE_VECS)      { bvl = bvl; nv = nr_vecs; }
	else if (nr_vecs)                   { bvl = bio->bi_inline_vecs; nv = BIO_INLINE_VECS; }
	else                                { bvl = NULL; nv = 0; }
	bio_init(bio, bdev, bvl, nv, opf);
	bio->bi_pool = bs;
	return bio;
}
EXPORT_SYMBOL(bio_alloc_bioset);

/* ======================= bio 释放 ======================= */
static void bio_alloc_cache_prune(struct bio_alloc_cache *cache, unsigned int nr)
{
	struct bio *bio;
	unsigned int i = 0;
	while ((bio = cache->free_list) && i < nr) {
		cache->free_list = bio->bi_next;
		cache->nr--;
		bio_free(bio);
		i++;
	}
}

void bio_put(struct bio *bio)
{
	if (unlikely(bio_flagged(bio, BIO_REFFED))) {
		BUG_ON(!atomic_read(&bio->__bi_cnt));
		if (!atomic_dec_and_test(&bio->__bi_cnt)) return;
	}
	if (bio->bi_opf & REQ_ALLOC_CACHE) {
		struct bio_alloc_cache *cache;
		bio_uninit(bio);
		cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
		bio->bi_next = cache->free_list;
		cache->free_list = bio;
		if (++cache->nr > ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK)
			bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK);
		put_cpu();
	} else bio_free(bio);
}
EXPORT_SYMBOL(bio_put);

/* ======================= bio 克隆与分裂 ======================= */
static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
{
	bio_set_flag(bio, BIO_CLONED);
	bio->bi_ioprio = bio_src->bi_ioprio;
	bio->bi_iter = bio_src->bi_iter;
	if (bio->bi_bdev) {
		if (bio->bi_bdev == bio_src->bi_bdev && bio_flagged(bio_src, BIO_REMAPPED))
			bio_set_flag(bio, BIO_REMAPPED);
		bio_clone_blkg_association(bio, bio_src);
	}
	if (bio_crypt_clone(bio, bio_src, gfp) < 0) return -ENOMEM;
	if (bio_integrity(bio_src) && bio_integrity_clone(bio, bio_src, gfp) < 0)
		return -ENOMEM;
	return 0;
}

struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
			    gfp_t gfp, struct bio_set *bs)
{
	struct bio *bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs);
	if (!bio) return NULL;
	if (__bio_clone(bio, bio_src, gfp) < 0) { bio_put(bio); return NULL; }
	bio->bi_io_vec = bio_src->bi_io_vec;
	return bio;
}
EXPORT_SYMBOL(bio_alloc_clone);

int bio_init_clone(struct block_device *bdev, struct bio *bio,
		   struct bio *bio_src, gfp_t gfp)
{
	int ret;
	bio_init(bio, bdev, bio_src->bi_io_vec, 0, bio_src->bi_opf);
	ret = __bio_clone(bio, bio_src, gfp);
	if (ret) bio_uninit(bio);
	return ret;
}
EXPORT_SYMBOL(bio_init_clone);

struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs)
{
	struct bio *split;
	BUG_ON(sectors <= 0 || sectors >= bio_sectors(bio));
	if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND)) return NULL;
	split = bio_alloc_clone(bio->bi_bdev, bio, gfp, bs);
	if (!split) return NULL;
	split->bi_iter.bi_size = sectors << 9;
	if (bio_integrity(split)) bio_integrity_trim(split);
	bio_advance(bio, split->bi_iter.bi_size);
	if (bio_flagged(bio, BIO_TRACE_COMPLETION))
		bio_set_flag(split, BIO_TRACE_COMPLETION);
	return split;
}
EXPORT_SYMBOL(bio_split);

/* ======================= bio 页操作 ======================= */
static inline bool bio_full(struct bio *bio, unsigned len)
{
	return bio->bi_vcnt >= bio->bi_max_vecs || bio->bi_iter.bi_size > UINT_MAX - len;
}

static bool page_is_mergeable(const struct bio_vec *bv,
		struct page *page, unsigned int len, unsigned int off, bool *same_page)
{
	size_t bv_end = bv->bv_offset + bv->bv_len;
	phys_addr_t vec_end = page_to_phys(bv->bv_page) + bv_end - 1;
	phys_addr_t page_addr = page_to_phys(page);

	if (vec_end + 1 != page_addr + off) return false;
	if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) return false;
	*same_page = ((vec_end & PAGE_MASK) == page_addr);
	if (*same_page) return true;
	return (bv->bv_page + bv_end / PAGE_SIZE) == (page + off / PAGE_SIZE);
}

static bool __bio_try_merge_page(struct bio *bio, struct page *page,
		unsigned int len, unsigned int off, bool *same_page)
{
	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return false;
	if (bio->bi_vcnt > 0) {
		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
		if (page_is_mergeable(bv, page, len, off, same_page)) {
			if (bio->bi_iter.bi_size > UINT_MAX - len) {
				*same_page = false; return false;
			}
			bv->bv_len += len;
			bio->bi_iter.bi_size += len;
			return true;
		}
	}
	return false;
}

void __bio_add_page(struct bio *bio, struct page *page,
		unsigned int len, unsigned int off)
{
	struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
	WARN_ON_ONCE(bio_full(bio, len));
	bv->bv_page = page; bv->bv_offset = off; bv->bv_len = len;
	bio->bi_iter.bi_size += len;
	bio->bi_vcnt++;
}
EXPORT_SYMBOL_GPL(__bio_add_page);

int bio_add_page(struct bio *bio, struct page *page,
		 unsigned int len, unsigned int offset)
{
	bool same_page = false;
	if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) {
		if (bio_full(bio, len)) return 0;
		__bio_add_page(bio, page, len, offset);
	}
	return len;
}
EXPORT_SYMBOL(bio_add_page);

/* 硬件约束合并校验 */
static bool bio_try_merge_hw_seg(struct request_queue *q, struct bio *bio,
		struct page *page, unsigned len, unsigned offset, bool *same)
{
	struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
	unsigned long mask = queue_segment_boundary(q);
	phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset;
	phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;

	if ((addr1 | mask) != (addr2 | mask)) return false;
	if (bv->bv_len + len > queue_max_segment_size(q)) return false;
	return __bio_try_merge_page(bio, page, len, offset, same);
}

int bio_add_hw_page(struct request_queue *q, struct bio *bio,
		struct page *page, unsigned int len, unsigned int offset,
		unsigned int max_sectors, bool *same_page)
{
	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0;
	if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) return 0;
	if (bio->bi_vcnt > 0) {
		if (bio_try_merge_hw_seg(q, bio, page, len, offset, same_page))
			return len;
		if (bvec_gap_to_prev(&q->limits, &bio->bi_io_vec[bio->bi_vcnt-1], offset))
			return 0;
	}
	if (bio_full(bio, len) || bio->bi_vcnt >= queue_max_segments(q))
		return 0;
	__bio_add_page(bio, page, len, offset);
	return len;
}
EXPORT_SYMBOL(bio_add_hw_page);

/* 通用 add_page / zone_append 公共逻辑 */
static int bio_add_page_or_hw(struct bio *bio, struct page *page,
		unsigned int len, unsigned int offset, bool zone_append)
{
	if (zone_append) {
		struct request_queue *q = bdev_get_queue(bio->bi_bdev);
		bool same_page = false;
		if (bio_add_hw_page(q, bio, page, len, offset,
				queue_max_zone_append_sectors(q), &same_page) != len)
			return -EINVAL;
		if (same_page) put_page(page);
	} else {
		bool same_page = false;
		if (!__bio_try_merge_page(bio, page, len, offset, &same_page))
			__bio_add_page(bio, page, len, offset);
		else if (same_page) put_page(page);
	}
	return 0;
}

/* ======================= iov 迭代器 -> bio 页 ======================= */
#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))

static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
{
	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
	struct page **pages = (struct page **)bv;
	ssize_t size, left;
	unsigned len, i = 0;
	size_t offset, trim;
	int ret = 0;

	pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
	size = iov_iter_get_pages2(iter, pages, UINT_MAX - bio->bi_iter.bi_size,
				  nr_pages, &offset);
	if (unlikely(size <= 0)) return size ? size : -EFAULT;
	nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE);
	trim = size & (bdev_logical_block_size(bio->bi_bdev) - 1);
	iov_iter_revert(iter, trim);
	size -= trim;
	if (unlikely(!size)) { ret = -EFAULT; goto out; }

	bool zone = (bio_op(bio) == REQ_OP_ZONE_APPEND);
	for (left = size, i = 0; left > 0; left -= len, i++) {
		struct page *page = pages[i];
		len = min_t(size_t, PAGE_SIZE - offset, left);
		ret = bio_add_page_or_hw(bio, page, len, offset, zone);
		if (ret) break;
		offset = 0;
	}
	iov_iter_revert(iter, left);
out:
	while (i < nr_pages) put_page(pages[i++]);
	return ret;
}

int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
{
	if (iov_iter_is_bvec(iter)) {
		bio_iov_bvec_set(bio, iter);
		iov_iter_advance(iter, bio->bi_iter.bi_size);
		return 0;
	}
	int ret;
	do {
		ret = __bio_iov_iter_get_pages(bio, iter);
	} while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
	return bio->bi_vcnt ? 0 : ret;
}
EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);

/* ======================= 提交等待 ======================= */
static void submit_bio_wait_endio(struct bio *bio) { complete(bio->bi_private); }

int submit_bio_wait(struct bio *bio)
{
	DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_bdev->bd_disk->lockdep_map);
	bio->bi_private = &done;
	bio->bi_end_io = submit_bio_wait_endio;
	bio->bi_opf |= REQ_SYNC;
	submit_bio(bio);
	unsigned long hang_check = sysctl_hung_task_timeout_secs;
	if (hang_check)
		while (!wait_for_completion_io_timeout(&done, hang_check * (HZ/2)));
	else
		wait_for_completion_io(&done);
	return blk_status_to_errno(bio->bi_status);
}
EXPORT_SYMBOL(submit_bio_wait);

/* ======================= 辅助：advance, copy, dirty ======================= */
void __bio_advance(struct bio *bio, unsigned bytes)
{
	if (bio_integrity(bio)) bio_integrity_advance(bio, bytes);
	bio_crypt_advance(bio, bytes);
	bio_advance_iter(bio, &bio->bi_iter, bytes);
}
EXPORT_SYMBOL(__bio_advance);

void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
			struct bio *src, struct bvec_iter *src_iter)
{
	while (src_iter->bi_size && dst_iter->bi_size) {
		struct bio_vec src_bv = bio_iter_iovec(src, *src_iter);
		struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter);
		unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len);
		void *src_buf = bvec_kmap_local(&src_bv);
		void *dst_buf = bvec_kmap_local(&dst_bv);
		memcpy(dst_buf, src_buf, bytes);
		kunmap_local(dst_buf);
		kunmap_local(src_buf);
		bio_advance_iter_single(src, src_iter, bytes);
		bio_advance_iter_single(dst, dst_iter, bytes);
	}
}
EXPORT_SYMBOL(bio_copy_data_iter);

void bio_copy_data(struct bio *dst, struct bio *src)
{
	struct bvec_iter src_iter = src->bi_iter;
	struct bvec_iter dst_iter = dst->bi_iter;
	bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
}
EXPORT_SYMBOL(bio_copy_data);

void bio_set_pages_dirty(struct bio *bio)
{
	struct bio_vec *bvec;
	struct bvec_iter_all iter_all;
	bio_for_each_segment_all(bvec, bio, iter_all)
		if (!PageCompound(bvec->bv_page))
			set_page_dirty_lock(bvec->bv_page);
}

static void bio_dirty_fn(struct work_struct *work);
static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
static DEFINE_SPINLOCK(bio_dirty_lock);
static struct bio *bio_dirty_list;

static void bio_dirty_fn(struct work_struct *work)
{
	struct bio *bio, *next;
	spin_lock_irq(&bio_dirty_lock);
	next = bio_dirty_list;
	bio_dirty_list = NULL;
	spin_unlock_irq(&bio_dirty_lock);
	while ((bio = next) != NULL) {
		next = bio->bi_private;
		bio_release_pages(bio, true);
		bio_put(bio);
	}
}

void bio_check_pages_dirty(struct bio *bio)
{
	struct bio_vec *bvec;
	struct bvec_iter_all iter_all;
	bio_for_each_segment_all(bvec, bio, iter_all)
		if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
			goto defer;
	bio_release_pages(bio, false);
	bio_put(bio);
	return;
defer:
	spin_lock_irqsave(&bio_dirty_lock, flags);
	bio->bi_private = bio_dirty_list;
	bio_dirty_list = bio;
	spin_unlock_irqrestore(&bio_dirty_lock, flags);
	schedule_work(&bio_dirty_work);
}

/* ======================= 结束 I/O ======================= */
static inline bool bio_remaining_done(struct bio *bio)
{
	if (!bio_flagged(bio, BIO_CHAIN)) return true;
	BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
	if (atomic_dec_and_test(&bio->__bi_remaining)) {
		bio_clear_flag(bio, BIO_CHAIN);
		return true;
	}
	return false;
}

void bio_endio(struct bio *bio)
{
again:
	if (!bio_remaining_done(bio)) return;
	if (!bio_integrity_endio(bio)) return;
	rq_qos_done_bio(bio);
	if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
		trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio);
		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
	}
	blk_throtl_bio_endio(bio);
	bio_uninit(bio);
	if (bio->bi_end_io) bio->bi_end_io(bio);
}
EXPORT_SYMBOL(bio_endio);

/* ======================= bio_set 生命周期 ======================= */
int biovec_init_pool(mempool_t *pool, int pool_entries)
{
	return mempool_init_slab_pool(pool, pool_entries,
				      bvec_slabs[ARRAY_SIZE(bvec_slabs)-1].slab);
}

static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node)
{
	struct bio_set *bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead);
	if (bs->cache) {
		struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);
		bio_alloc_cache_prune(cache, -1U);
	}
	return 0;
}

static void bio_alloc_cache_destroy(struct bio_set *bs)
{
	int cpu;
	if (!bs->cache) return;
	cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
	for_each_possible_cpu(cpu) {
		struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);
		bio_alloc_cache_prune(cache, -1U);
	}
	free_percpu(bs->cache);
	bs->cache = NULL;
}

void bioset_exit(struct bio_set *bs)
{
	bio_alloc_cache_destroy(bs);
	if (bs->rescue_workqueue) destroy_workqueue(bs->rescue_workqueue);
	bs->rescue_workqueue = NULL;
	mempool_exit(&bs->bio_pool);
	mempool_exit(&bs->bvec_pool);
	bioset_integrity_free(bs);
	if (bs->bio_slab) bio_put_slab(bs);
	bs->bio_slab = NULL;
}
EXPORT_SYMBOL(bioset_exit);

int bioset_init(struct bio_set *bs, unsigned int pool_size,
		unsigned int front_pad, int flags)
{
	bs->front_pad = front_pad;
	bs->back_pad = (flags & BIOSET_NEED_BVECS) ?
		BIO_INLINE_VECS * sizeof(struct bio_vec) : 0;
	spin_lock_init(&bs->rescue_lock);
	bio_list_init(&bs->rescue_list);
	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);

	bs->bio_slab = bio_find_or_create_slab(bs);
	if (!bs->bio_slab) return -ENOMEM;
	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
		goto bad;
	if ((flags & BIOSET_NEED_BVECS) && biovec_init_pool(&bs->bvec_pool, pool_size))
		goto bad;
	if (flags & BIOSET_NEED_RESCUER) {
		bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
		if (!bs->rescue_workqueue) goto bad;
	}
	if (flags & BIOSET_PERCPU_CACHE) {
		bs->cache = alloc_percpu(struct bio_alloc_cache);
		if (!bs->cache) goto bad;
		cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
	}
	return 0;
bad:
	bioset_exit(bs);
	return -ENOMEM;
}
EXPORT_SYMBOL(bioset_init);

/* ======================= 模块初始化 ======================= */
static int __init init_bio(void)
{
	int i;
	bio_integrity_init();
	for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
		struct biovec_slab *bvs = &bvec_slabs[i];
		bvs->slab = kmem_cache_create(bvs->name,
				bvs->nr_vecs * sizeof(struct bio_vec), 0,
				SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
	}
	cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, bio_cpu_dead);
	if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0,
			BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE))
		panic("bio: can't allocate bios\n");
	if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
		panic("bio: can't create integrity pool\n");
	return 0;
}
subsys_initcall(init_bio);

这是九章编程法的巨大作用之一，不只是编制程序，而是对既有运行多年的程序，可以进行再组织重构，有效进行冗余和重复的精准合并，压缩，大幅降低程序的可执行性，可维护性，提升运行效率和降低能耗。程序不是没有错就完美，而是要让其达到极限的简洁，而功能和计算更准确，才是最合理的程序。庞大的既有程序库，包括运行几十年的无BUG程序，也是大量的重复和冗余。

接近 73.5% 的压缩率。代码从 1700 行压缩至 450 行，外部接口完全保留，所有结构重复已消除。每一个逻辑原语在系统中只出现一次，达到此模块的物理极限。此程序是重构的，与原程序一样，但是压缩了大量的重复代码。大量的老系统程序或是新开发程序，实际需要，或是有用的并不多，大理是重复的引用，验证，或是循环线形嵌套结构，让代码量暴发式增长，这也是巨大的能耗和性能降低。

九章编程法，不但能编写新程序，转换不同编程语言，更能对老旧系统程序进行重新组织重构，功能不变，程序精细化，消除重复冗余，让程序更好用，更好管，更好扩展应用。降能耗，提性能，清除BUG。

我们看到一个成熟的系统，往往数成千，几亿行代码，往往百分之七八十是冗余和复杂的代码，吃掉了大量的计算性能和存储性能。屎山代码，并不冤。大理商业企业往往以代码量来表达系统的成熟度，实际上，也表达了另一个意思，代码吃掉了同样的性能以及存大巨量的BUG。