1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2017 Western Digital Corporation or its affiliates.
4	*
5	* This file is released under the GPL.
6	*/
7
8	#include "dm-zoned.h"
9
10	#include <linux/module.h>
11	#include <linux/crc32.h>
12	#include <linux/sched/mm.h>
13
14	#define DM_MSG_PREFIX "zoned metadata"
15
16	/*
17	* Metadata version.
18	*/
19	#define DMZ_META_VER 2
20
21	/*
22	* On-disk super block magic.
23	*/
24	#define DMZ_MAGIC ((((unsigned int)('D')) << 24) | \
25	(((unsigned int)('Z')) << 16) | \
26	(((unsigned int)('B')) << 8) | \
27	((unsigned int)('D')))
28
29	/*
30	* On disk super block.
31	* This uses only 512 B but uses on disk a full 4KB block. This block is
32	* followed on disk by the mapping table of chunks to zones and the bitmap
33	* blocks indicating zone block validity.
34	* The overall resulting metadata format is:
35	* (1) Super block (1 block)
36	* (2) Chunk mapping table (nr_map_blocks)
37	* (3) Bitmap blocks (nr_bitmap_blocks)
38	* All metadata blocks are stored in conventional zones, starting from
39	* the first conventional zone found on disk.
40	*/
41	struct dmz_super {
42	/* Magic number */
43	__le32 magic; /* 4 */
44
45	/* Metadata version number */
46	__le32 version; /* 8 */
47
48	/* Generation number */
49	__le64 gen; /* 16 */
50
51	/* This block number */
52	__le64 sb_block; /* 24 */
53
54	/* The number of metadata blocks, including this super block */
55	__le32 nr_meta_blocks; /* 28 */
56
57	/* The number of sequential zones reserved for reclaim */
58	__le32 nr_reserved_seq; /* 32 */
59
60	/* The number of entries in the mapping table */
61	__le32 nr_chunks; /* 36 */
62
63	/* The number of blocks used for the chunk mapping table */
64	__le32 nr_map_blocks; /* 40 */
65
66	/* The number of blocks used for the block bitmaps */
67	__le32 nr_bitmap_blocks; /* 44 */
68
69	/* Checksum */
70	__le32 crc; /* 48 */
71
72	/* DM-Zoned label */
73	u8 dmz_label[32]; /* 80 */
74
75	/* DM-Zoned UUID */
76	u8 dmz_uuid[16]; /* 96 */
77
78	/* Device UUID */
79	u8 dev_uuid[16]; /* 112 */
80
81	/* Padding to full 512B sector */
82	u8 reserved[400]; /* 512 */
83	};
84
85	/*
86	* Chunk mapping entry: entries are indexed by chunk number
87	* and give the zone ID (dzone_id) mapping the chunk on disk.
88	* This zone may be sequential or random. If it is a sequential
89	* zone, a second zone (bzone_id) used as a write buffer may
90	* also be specified. This second zone will always be a randomly
91	* writeable zone.
92	*/
93	struct dmz_map {
94	__le32 dzone_id;
95	__le32 bzone_id;
96	};
97
98	/*
99	* Chunk mapping table metadata: 512 8-bytes entries per 4KB block.
100	*/
101	#define DMZ_MAP_ENTRIES (DMZ_BLOCK_SIZE / sizeof(struct dmz_map))
102	#define DMZ_MAP_ENTRIES_SHIFT (ilog2(DMZ_MAP_ENTRIES))
103	#define DMZ_MAP_ENTRIES_MASK (DMZ_MAP_ENTRIES - 1)
104	#define DMZ_MAP_UNMAPPED UINT_MAX
105
106	/*
107	* Meta data block descriptor (for cached metadata blocks).
108	*/
109	struct dmz_mblock {
110	struct rb_node node;
111	struct list_head link;
112	sector_t no;
113	unsigned int ref;
114	unsigned long state;
115	struct page *page;
116	void *data;
117	};
118
119	/*
120	* Metadata block state flags.
121	*/
122	enum {
123	DMZ_META_DIRTY,
124	DMZ_META_READING,
125	DMZ_META_WRITING,
126	DMZ_META_ERROR,
127	};
128
129	/*
130	* Super block information (one per metadata set).
131	*/
132	struct dmz_sb {
133	sector_t block;
134	struct dmz_dev *dev;
135	struct dmz_mblock *mblk;
136	struct dmz_super *sb;
137	struct dm_zone *zone;
138	};
139
140	/*
141	* In-memory metadata.
142	*/
143	struct dmz_metadata {
144	struct dmz_dev *dev;
145	unsigned int nr_devs;
146
147	char devname[BDEVNAME_SIZE];
148	char label[BDEVNAME_SIZE];
149	uuid_t uuid;
150
151	sector_t zone_bitmap_size;
152	unsigned int zone_nr_bitmap_blocks;
153	unsigned int zone_bits_per_mblk;
154
155	sector_t zone_nr_blocks;
156	sector_t zone_nr_blocks_shift;
157
158	sector_t zone_nr_sectors;
159	sector_t zone_nr_sectors_shift;
160
161	unsigned int nr_bitmap_blocks;
162	unsigned int nr_map_blocks;
163
164	unsigned int nr_zones;
165	unsigned int nr_useable_zones;
166	unsigned int nr_meta_blocks;
167	unsigned int nr_meta_zones;
168	unsigned int nr_data_zones;
169	unsigned int nr_cache_zones;
170	unsigned int nr_rnd_zones;
171	unsigned int nr_reserved_seq;
172	unsigned int nr_chunks;
173
174	/* Zone information array */
175	struct xarray zones;
176
177	struct dmz_sb sb[2];
178	unsigned int mblk_primary;
179	unsigned int sb_version;
180	u64 sb_gen;
181	unsigned int min_nr_mblks;
182	unsigned int max_nr_mblks;
183	atomic_t nr_mblks;
184	struct rw_semaphore mblk_sem;
185	struct mutex mblk_flush_lock;
186	spinlock_t mblk_lock;
187	struct rb_root mblk_rbtree;
188	struct list_head mblk_lru_list;
189	struct list_head mblk_dirty_list;
190	struct shrinker *mblk_shrinker;
191
192	/* Zone allocation management */
193	struct mutex map_lock;
194	struct dmz_mblock **map_mblk;
195
196	unsigned int nr_cache;
197	atomic_t unmap_nr_cache;
198	struct list_head unmap_cache_list;
199	struct list_head map_cache_list;
200
201	atomic_t nr_reserved_seq_zones;
202	struct list_head reserved_seq_zones_list;
203
204	wait_queue_head_t free_wq;
205	};
206
207	#define dmz_zmd_info(zmd, format, args...) \
208	DMINFO("(%s): " format, (zmd)->label, ## args)
209
210	#define dmz_zmd_err(zmd, format, args...) \
211	DMERR("(%s): " format, (zmd)->label, ## args)
212
213	#define dmz_zmd_warn(zmd, format, args...) \
214	DMWARN("(%s): " format, (zmd)->label, ## args)
215
216	#define dmz_zmd_debug(zmd, format, args...) \
217	DMDEBUG("(%s): " format, (zmd)->label, ## args)
218	/*
219	* Various accessors
220	*/
221	static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)
222	{
223	if (WARN_ON(!zone))
224	return 0;
225
226	return zone->id - zone->dev->zone_offset;
227	}
228
229	sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)
230	{
231	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
232
233	return (sector_t)zone_id << zmd->zone_nr_sectors_shift;
234	}
235
236	sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)
237	{
238	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
239
240	return (sector_t)zone_id << zmd->zone_nr_blocks_shift;
241	}
242
243	unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)
244	{
245	return zmd->zone_nr_blocks;
246	}
247
248	unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)
249	{
250	return zmd->zone_nr_sectors;
251	}
252
253	unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)
254	{
255	return zmd->zone_nr_sectors_shift;
256	}
257
258	unsigned int dmz_nr_zones(struct dmz_metadata *zmd)
259	{
260	return zmd->nr_zones;
261	}
262
263	unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)
264	{
265	return zmd->nr_chunks;
266	}
267
268	unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)
269	{
270	return zmd->dev[idx].nr_rnd;
271	}
272
273	unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)
274	{
275	return atomic_read(v: &zmd->dev[idx].unmap_nr_rnd);
276	}
277
278	unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)
279	{
280	return zmd->nr_cache;
281	}
282
283	unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)
284	{
285	return atomic_read(v: &zmd->unmap_nr_cache);
286	}
287
288	unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)
289	{
290	return zmd->dev[idx].nr_seq;
291	}
292
293	unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)
294	{
295	return atomic_read(v: &zmd->dev[idx].unmap_nr_seq);
296	}
297
298	static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)
299	{
300	return xa_load(&zmd->zones, index: zone_id);
301	}
302
303	static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,
304	unsigned int zone_id, struct dmz_dev *dev)
305	{
306	struct dm_zone *zone = kzalloc(sizeof(struct dm_zone), GFP_KERNEL);
307
308	if (!zone)
309	return ERR_PTR(error: -ENOMEM);
310
311	if (xa_insert(xa: &zmd->zones, index: zone_id, entry: zone, GFP_KERNEL)) {
312	kfree(objp: zone);
313	return ERR_PTR(error: -EBUSY);
314	}
315
316	INIT_LIST_HEAD(list: &zone->link);
317	atomic_set(v: &zone->refcount, i: 0);
318	zone->id = zone_id;
319	zone->chunk = DMZ_MAP_UNMAPPED;
320	zone->dev = dev;
321
322	return zone;
323	}
324
325	const char *dmz_metadata_label(struct dmz_metadata *zmd)
326	{
327	return (const char *)zmd->label;
328	}
329
330	bool dmz_check_dev(struct dmz_metadata *zmd)
331	{
332	unsigned int i;
333
334	for (i = 0; i < zmd->nr_devs; i++) {
335	if (!dmz_check_bdev(dmz_dev: &zmd->dev[i]))
336	return false;
337	}
338	return true;
339	}
340
341	bool dmz_dev_is_dying(struct dmz_metadata *zmd)
342	{
343	unsigned int i;
344
345	for (i = 0; i < zmd->nr_devs; i++) {
346	if (dmz_bdev_is_dying(dmz_dev: &zmd->dev[i]))
347	return true;
348	}
349	return false;
350	}
351
352	/*
353	* Lock/unlock mapping table.
354	* The map lock also protects all the zone lists.
355	*/
356	void dmz_lock_map(struct dmz_metadata *zmd)
357	{
358	mutex_lock(&zmd->map_lock);
359	}
360
361	void dmz_unlock_map(struct dmz_metadata *zmd)
362	{
363	mutex_unlock(lock: &zmd->map_lock);
364	}
365
366	/*
367	* Lock/unlock metadata access. This is a "read" lock on a semaphore
368	* that prevents metadata flush from running while metadata are being
369	* modified. The actual metadata write mutual exclusion is achieved with
370	* the map lock and zone state management (active and reclaim state are
371	* mutually exclusive).
372	*/
373	void dmz_lock_metadata(struct dmz_metadata *zmd)
374	{
375	down_read(sem: &zmd->mblk_sem);
376	}
377
378	void dmz_unlock_metadata(struct dmz_metadata *zmd)
379	{
380	up_read(sem: &zmd->mblk_sem);
381	}
382
383	/*
384	* Lock/unlock flush: prevent concurrent executions
385	* of dmz_flush_metadata as well as metadata modification in reclaim
386	* while flush is being executed.
387	*/
388	void dmz_lock_flush(struct dmz_metadata *zmd)
389	{
390	mutex_lock(&zmd->mblk_flush_lock);
391	}
392
393	void dmz_unlock_flush(struct dmz_metadata *zmd)
394	{
395	mutex_unlock(lock: &zmd->mblk_flush_lock);
396	}
397
398	/*
399	* Allocate a metadata block.
400	*/
401	static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,
402	sector_t mblk_no)
403	{
404	struct dmz_mblock *mblk = NULL;
405
406	/* See if we can reuse cached blocks */
407	if (zmd->max_nr_mblks && atomic_read(v: &zmd->nr_mblks) > zmd->max_nr_mblks) {
408	spin_lock(lock: &zmd->mblk_lock);
409	mblk = list_first_entry_or_null(&zmd->mblk_lru_list,
410	struct dmz_mblock, link);
411	if (mblk) {
412	list_del_init(entry: &mblk->link);
413	rb_erase(&mblk->node, &zmd->mblk_rbtree);
414	mblk->no = mblk_no;
415	}
416	spin_unlock(lock: &zmd->mblk_lock);
417	if (mblk)
418	return mblk;
419	}
420
421	/* Allocate a new block */
422	mblk = kmalloc(sizeof(struct dmz_mblock), GFP_NOIO);
423	if (!mblk)
424	return NULL;
425
426	mblk->page = alloc_page(GFP_NOIO);
427	if (!mblk->page) {
428	kfree(objp: mblk);
429	return NULL;
430	}
431
432	RB_CLEAR_NODE(&mblk->node);
433	INIT_LIST_HEAD(list: &mblk->link);
434	mblk->ref = 0;
435	mblk->state = 0;
436	mblk->no = mblk_no;
437	mblk->data = page_address(mblk->page);
438
439	atomic_inc(v: &zmd->nr_mblks);
440
441	return mblk;
442	}
443
444	/*
445	* Free a metadata block.
446	*/
447	static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
448	{
449	__free_pages(page: mblk->page, order: 0);
450	kfree(objp: mblk);
451
452	atomic_dec(v: &zmd->nr_mblks);
453	}
454
455	/*
456	* Insert a metadata block in the rbtree.
457	*/
458	static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
459	{
460	struct rb_root *root = &zmd->mblk_rbtree;
461	struct rb_node **new = &(root->rb_node), *parent = NULL;
462	struct dmz_mblock *b;
463
464	/* Figure out where to put the new node */
465	while (*new) {
466	b = container_of(*new, struct dmz_mblock, node);
467	parent = *new;
468	new = (b->no < mblk->no) ? &((*new)->rb_left) : &((*new)->rb_right);
469	}
470
471	/* Add new node and rebalance tree */
472	rb_link_node(node: &mblk->node, parent, rb_link: new);
473	rb_insert_color(&mblk->node, root);
474	}
475
476	/*
477	* Lookup a metadata block in the rbtree. If the block is found, increment
478	* its reference count.
479	*/
480	static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,
481	sector_t mblk_no)
482	{
483	struct rb_root *root = &zmd->mblk_rbtree;
484	struct rb_node *node = root->rb_node;
485	struct dmz_mblock *mblk;
486
487	while (node) {
488	mblk = container_of(node, struct dmz_mblock, node);
489	if (mblk->no == mblk_no) {
490	/*
491	* If this is the first reference to the block,
492	* remove it from the LRU list.
493	*/
494	mblk->ref++;
495	if (mblk->ref == 1 &&
496	!test_bit(DMZ_META_DIRTY, &mblk->state))
497	list_del_init(entry: &mblk->link);
498	return mblk;
499	}
500	node = (mblk->no < mblk_no) ? node->rb_left : node->rb_right;
501	}
502
503	return NULL;
504	}
505
506	/*
507	* Metadata block BIO end callback.
508	*/
509	static void dmz_mblock_bio_end_io(struct bio *bio)
510	{
511	struct dmz_mblock *mblk = bio->bi_private;
512	int flag;
513
514	if (bio->bi_status)
515	set_bit(nr: DMZ_META_ERROR, addr: &mblk->state);
516
517	if (bio_op(bio) == REQ_OP_WRITE)
518	flag = DMZ_META_WRITING;
519	else
520	flag = DMZ_META_READING;
521
522	clear_bit_unlock(nr: flag, addr: &mblk->state);
523	smp_mb__after_atomic();
524	wake_up_bit(word: &mblk->state, bit: flag);
525
526	bio_put(bio);
527	}
528
529	/*
530	* Read an uncached metadata block from disk and add it to the cache.
531	*/
532	static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,
533	sector_t mblk_no)
534	{
535	struct dmz_mblock *mblk, *m;
536	sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;
537	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
538	struct bio *bio;
539
540	if (dmz_bdev_is_dying(dmz_dev: dev))
541	return ERR_PTR(error: -EIO);
542
543	/* Get a new block and a BIO to read it */
544	mblk = dmz_alloc_mblock(zmd, mblk_no);
545	if (!mblk)
546	return ERR_PTR(error: -ENOMEM);
547
548	bio = bio_alloc(bdev: dev->bdev, nr_vecs: 1, opf: REQ_OP_READ | REQ_META | REQ_PRIO,
549	GFP_NOIO);
550
551	spin_lock(lock: &zmd->mblk_lock);
552
553	/*
554	* Make sure that another context did not start reading
555	* the block already.
556	*/
557	m = dmz_get_mblock_fast(zmd, mblk_no);
558	if (m) {
559	spin_unlock(lock: &zmd->mblk_lock);
560	dmz_free_mblock(zmd, mblk);
561	bio_put(bio);
562	return m;
563	}
564
565	mblk->ref++;
566	set_bit(nr: DMZ_META_READING, addr: &mblk->state);
567	dmz_insert_mblock(zmd, mblk);
568
569	spin_unlock(lock: &zmd->mblk_lock);
570
571	/* Submit read BIO */
572	bio->bi_iter.bi_sector = dmz_blk2sect(block);
573	bio->bi_private = mblk;
574	bio->bi_end_io = dmz_mblock_bio_end_io;
575	__bio_add_page(bio, page: mblk->page, DMZ_BLOCK_SIZE, off: 0);
576	submit_bio(bio);
577
578	return mblk;
579	}
580
581	/*
582	* Free metadata blocks.
583	*/
584	static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,
585	unsigned long limit)
586	{
587	struct dmz_mblock *mblk;
588	unsigned long count = 0;
589
590	if (!zmd->max_nr_mblks)
591	return 0;
592
593	while (!list_empty(head: &zmd->mblk_lru_list) &&
594	atomic_read(v: &zmd->nr_mblks) > zmd->min_nr_mblks &&
595	count < limit) {
596	mblk = list_first_entry(&zmd->mblk_lru_list,
597	struct dmz_mblock, link);
598	list_del_init(entry: &mblk->link);
599	rb_erase(&mblk->node, &zmd->mblk_rbtree);
600	dmz_free_mblock(zmd, mblk);
601	count++;
602	}
603
604	return count;
605	}
606
607	/*
608	* For mblock shrinker: get the number of unused metadata blocks in the cache.
609	*/
610	static unsigned long dmz_mblock_shrinker_count(struct shrinker *shrink,
611	struct shrink_control *sc)
612	{
613	struct dmz_metadata *zmd = shrink->private_data;
614
615	return atomic_read(v: &zmd->nr_mblks);
616	}
617
618	/*
619	* For mblock shrinker: scan unused metadata blocks and shrink the cache.
620	*/
621	static unsigned long dmz_mblock_shrinker_scan(struct shrinker *shrink,
622	struct shrink_control *sc)
623	{
624	struct dmz_metadata *zmd = shrink->private_data;
625	unsigned long count;
626
627	spin_lock(lock: &zmd->mblk_lock);
628	count = dmz_shrink_mblock_cache(zmd, limit: sc->nr_to_scan);
629	spin_unlock(lock: &zmd->mblk_lock);
630
631	return count ? count : SHRINK_STOP;
632	}
633
634	/*
635	* Release a metadata block.
636	*/
637	static void dmz_release_mblock(struct dmz_metadata *zmd,
638	struct dmz_mblock *mblk)
639	{
640
641	if (!mblk)
642	return;
643
644	spin_lock(lock: &zmd->mblk_lock);
645
646	mblk->ref--;
647	if (mblk->ref == 0) {
648	if (test_bit(DMZ_META_ERROR, &mblk->state)) {
649	rb_erase(&mblk->node, &zmd->mblk_rbtree);
650	dmz_free_mblock(zmd, mblk);
651	} else if (!test_bit(DMZ_META_DIRTY, &mblk->state)) {
652	list_add_tail(new: &mblk->link, head: &zmd->mblk_lru_list);
653	dmz_shrink_mblock_cache(zmd, limit: 1);
654	}
655	}
656
657	spin_unlock(lock: &zmd->mblk_lock);
658	}
659
660	/*
661	* Get a metadata block from the rbtree. If the block
662	* is not present, read it from disk.
663	*/
664	static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,
665	sector_t mblk_no)
666	{
667	struct dmz_mblock *mblk;
668	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
669
670	/* Check rbtree */
671	spin_lock(lock: &zmd->mblk_lock);
672	mblk = dmz_get_mblock_fast(zmd, mblk_no);
673	spin_unlock(lock: &zmd->mblk_lock);
674
675	if (!mblk) {
676	/* Cache miss: read the block from disk */
677	mblk = dmz_get_mblock_slow(zmd, mblk_no);
678	if (IS_ERR(ptr: mblk))
679	return mblk;
680	}
681
682	/* Wait for on-going read I/O and check for error */
683	wait_on_bit_io(word: &mblk->state, bit: DMZ_META_READING,
684	TASK_UNINTERRUPTIBLE);
685	if (test_bit(DMZ_META_ERROR, &mblk->state)) {
686	dmz_release_mblock(zmd, mblk);
687	dmz_check_bdev(dmz_dev: dev);
688	return ERR_PTR(error: -EIO);
689	}
690
691	return mblk;
692	}
693
694	/*
695	* Mark a metadata block dirty.
696	*/
697	static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
698	{
699	spin_lock(lock: &zmd->mblk_lock);
700	if (!test_and_set_bit(nr: DMZ_META_DIRTY, addr: &mblk->state))
701	list_add_tail(new: &mblk->link, head: &zmd->mblk_dirty_list);
702	spin_unlock(lock: &zmd->mblk_lock);
703	}
704
705	/*
706	* Issue a metadata block write BIO.
707	*/
708	static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
709	unsigned int set)
710	{
711	struct dmz_dev *dev = zmd->sb[set].dev;
712	sector_t block = zmd->sb[set].block + mblk->no;
713	struct bio *bio;
714
715	if (dmz_bdev_is_dying(dmz_dev: dev))
716	return -EIO;
717
718	bio = bio_alloc(bdev: dev->bdev, nr_vecs: 1, opf: REQ_OP_WRITE | REQ_META | REQ_PRIO,
719	GFP_NOIO);
720
721	set_bit(nr: DMZ_META_WRITING, addr: &mblk->state);
722
723	bio->bi_iter.bi_sector = dmz_blk2sect(block);
724	bio->bi_private = mblk;
725	bio->bi_end_io = dmz_mblock_bio_end_io;
726	__bio_add_page(bio, page: mblk->page, DMZ_BLOCK_SIZE, off: 0);
727	submit_bio(bio);
728
729	return 0;
730	}
731
732	/*
733	* Read/write a metadata block.
734	*/
735	static int dmz_rdwr_block(struct dmz_dev *dev, enum req_op op,
736	sector_t block, struct page *page)
737	{
738	struct bio *bio;
739	int ret;
740
741	if (WARN_ON(!dev))
742	return -EIO;
743
744	if (dmz_bdev_is_dying(dmz_dev: dev))
745	return -EIO;
746
747	bio = bio_alloc(bdev: dev->bdev, nr_vecs: 1, opf: op | REQ_SYNC | REQ_META | REQ_PRIO,
748	GFP_NOIO);
749	bio->bi_iter.bi_sector = dmz_blk2sect(block);
750	__bio_add_page(bio, page, DMZ_BLOCK_SIZE, off: 0);
751	ret = submit_bio_wait(bio);
752	bio_put(bio);
753
754	if (ret)
755	dmz_check_bdev(dmz_dev: dev);
756	return ret;
757	}
758
759	/*
760	* Write super block of the specified metadata set.
761	*/
762	static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)
763	{
764	struct dmz_mblock *mblk = zmd->sb[set].mblk;
765	struct dmz_super *sb = zmd->sb[set].sb;
766	struct dmz_dev *dev = zmd->sb[set].dev;
767	sector_t sb_block;
768	u64 sb_gen = zmd->sb_gen + 1;
769	int ret;
770
771	sb->magic = cpu_to_le32(DMZ_MAGIC);
772
773	sb->version = cpu_to_le32(zmd->sb_version);
774	if (zmd->sb_version > 1) {
775	BUILD_BUG_ON(UUID_SIZE != 16);
776	export_uuid(dst: sb->dmz_uuid, src: &zmd->uuid);
777	memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);
778	export_uuid(dst: sb->dev_uuid, src: &dev->uuid);
779	}
780
781	sb->gen = cpu_to_le64(sb_gen);
782
783	/*
784	* The metadata always references the absolute block address,
785	* ie relative to the entire block range, not the per-device
786	* block address.
787	*/
788	sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;
789	sb->sb_block = cpu_to_le64(sb_block);
790	sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);
791	sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);
792	sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);
793
794	sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);
795	sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);
796
797	sb->crc = 0;
798	sb->crc = cpu_to_le32(crc32_le(sb_gen, (unsigned char *)sb, DMZ_BLOCK_SIZE));
799
800	ret = dmz_rdwr_block(dev, op: REQ_OP_WRITE, block: zmd->sb[set].block,
801	page: mblk->page);
802	if (ret == 0)
803	ret = blkdev_issue_flush(bdev: dev->bdev);
804
805	return ret;
806	}
807
808	/*
809	* Write dirty metadata blocks to the specified set.
810	*/
811	static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
812	struct list_head *write_list,
813	unsigned int set)
814	{
815	struct dmz_mblock *mblk;
816	struct dmz_dev *dev = zmd->sb[set].dev;
817	struct blk_plug plug;
818	int ret = 0, nr_mblks_submitted = 0;
819
820	/* Issue writes */
821	blk_start_plug(&plug);
822	list_for_each_entry(mblk, write_list, link) {
823	ret = dmz_write_mblock(zmd, mblk, set);
824	if (ret)
825	break;
826	nr_mblks_submitted++;
827	}
828	blk_finish_plug(&plug);
829
830	/* Wait for completion */
831	list_for_each_entry(mblk, write_list, link) {
832	if (!nr_mblks_submitted)
833	break;
834	wait_on_bit_io(word: &mblk->state, bit: DMZ_META_WRITING,
835	TASK_UNINTERRUPTIBLE);
836	if (test_bit(DMZ_META_ERROR, &mblk->state)) {
837	clear_bit(nr: DMZ_META_ERROR, addr: &mblk->state);
838	dmz_check_bdev(dmz_dev: dev);
839	ret = -EIO;
840	}
841	nr_mblks_submitted--;
842	}
843
844	/* Flush drive cache (this will also sync data) */
845	if (ret == 0)
846	ret = blkdev_issue_flush(bdev: dev->bdev);
847
848	return ret;
849	}
850
851	/*
852	* Log dirty metadata blocks.
853	*/
854	static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,
855	struct list_head *write_list)
856	{
857	unsigned int log_set = zmd->mblk_primary ^ 0x1;
858	int ret;
859
860	/* Write dirty blocks to the log */
861	ret = dmz_write_dirty_mblocks(zmd, write_list, set: log_set);
862	if (ret)
863	return ret;
864
865	/*
866	* No error so far: now validate the log by updating the
867	* log index super block generation.
868	*/
869	ret = dmz_write_sb(zmd, set: log_set);
870	if (ret)
871	return ret;
872
873	return 0;
874	}
875
876	/*
877	* Flush dirty metadata blocks.
878	*/
879	int dmz_flush_metadata(struct dmz_metadata *zmd)
880	{
881	struct dmz_mblock *mblk;
882	struct list_head write_list;
883	struct dmz_dev *dev;
884	int ret;
885
886	if (WARN_ON(!zmd))
887	return 0;
888
889	INIT_LIST_HEAD(list: &write_list);
890
891	/*
892	* Make sure that metadata blocks are stable before logging: take
893	* the write lock on the metadata semaphore to prevent target BIOs
894	* from modifying metadata.
895	*/
896	down_write(sem: &zmd->mblk_sem);
897	dev = zmd->sb[zmd->mblk_primary].dev;
898
899	/*
900	* This is called from the target flush work and reclaim work.
901	* Concurrent execution is not allowed.
902	*/
903	dmz_lock_flush(zmd);
904
905	if (dmz_bdev_is_dying(dmz_dev: dev)) {
906	ret = -EIO;
907	goto out;
908	}
909
910	/* Get dirty blocks */
911	spin_lock(lock: &zmd->mblk_lock);
912	list_splice_init(list: &zmd->mblk_dirty_list, head: &write_list);
913	spin_unlock(lock: &zmd->mblk_lock);
914
915	/* If there are no dirty metadata blocks, just flush the device cache */
916	if (list_empty(head: &write_list)) {
917	ret = blkdev_issue_flush(bdev: dev->bdev);
918	goto err;
919	}
920
921	/*
922	* The primary metadata set is still clean. Keep it this way until
923	* all updates are successful in the secondary set. That is, use
924	* the secondary set as a log.
925	*/
926	ret = dmz_log_dirty_mblocks(zmd, write_list: &write_list);
927	if (ret)
928	goto err;
929
930	/*
931	* The log is on disk. It is now safe to update in place
932	* in the primary metadata set.
933	*/
934	ret = dmz_write_dirty_mblocks(zmd, write_list: &write_list, set: zmd->mblk_primary);
935	if (ret)
936	goto err;
937
938	ret = dmz_write_sb(zmd, set: zmd->mblk_primary);
939	if (ret)
940	goto err;
941
942	while (!list_empty(head: &write_list)) {
943	mblk = list_first_entry(&write_list, struct dmz_mblock, link);
944	list_del_init(entry: &mblk->link);
945
946	spin_lock(lock: &zmd->mblk_lock);
947	clear_bit(nr: DMZ_META_DIRTY, addr: &mblk->state);
948	if (mblk->ref == 0)
949	list_add_tail(new: &mblk->link, head: &zmd->mblk_lru_list);
950	spin_unlock(lock: &zmd->mblk_lock);
951	}
952
953	zmd->sb_gen++;
954	out:
955	dmz_unlock_flush(zmd);
956	up_write(sem: &zmd->mblk_sem);
957
958	return ret;
959
960	err:
961	if (!list_empty(head: &write_list)) {
962	spin_lock(lock: &zmd->mblk_lock);
963	list_splice(list: &write_list, head: &zmd->mblk_dirty_list);
964	spin_unlock(lock: &zmd->mblk_lock);
965	}
966	if (!dmz_check_bdev(dmz_dev: dev))
967	ret = -EIO;
968	goto out;
969	}
970
971	/*
972	* Check super block.
973	*/
974	static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,
975	bool tertiary)
976	{
977	struct dmz_super *sb = dsb->sb;
978	struct dmz_dev *dev = dsb->dev;
979	unsigned int nr_meta_zones, nr_data_zones;
980	u32 crc, stored_crc;
981	u64 gen, sb_block;
982
983	if (le32_to_cpu(sb->magic) != DMZ_MAGIC) {
984	dmz_dev_err(dev, "Invalid meta magic (needed 0x%08x, got 0x%08x)",
985	DMZ_MAGIC, le32_to_cpu(sb->magic));
986	return -ENXIO;
987	}
988
989	zmd->sb_version = le32_to_cpu(sb->version);
990	if (zmd->sb_version > DMZ_META_VER) {
991	dmz_dev_err(dev, "Invalid meta version (needed %d, got %d)",
992	DMZ_META_VER, zmd->sb_version);
993	return -EINVAL;
994	}
995	if (zmd->sb_version < 2 && tertiary) {
996	dmz_dev_err(dev, "Tertiary superblocks are not supported");
997	return -EINVAL;
998	}
999
1000	gen = le64_to_cpu(sb->gen);
1001	stored_crc = le32_to_cpu(sb->crc);
1002	sb->crc = 0;
1003	crc = crc32_le(crc: gen, p: (unsigned char *)sb, DMZ_BLOCK_SIZE);
1004	if (crc != stored_crc) {
1005	dmz_dev_err(dev, "Invalid checksum (needed 0x%08x, got 0x%08x)",
1006	crc, stored_crc);
1007	return -ENXIO;
1008	}
1009
1010	sb_block = le64_to_cpu(sb->sb_block);
1011	if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift) {
1012	dmz_dev_err(dev, "Invalid superblock position (is %llu expected %llu)",
1013	sb_block, (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);
1014	return -EINVAL;
1015	}
1016	if (zmd->sb_version > 1) {
1017	uuid_t sb_uuid;
1018
1019	import_uuid(dst: &sb_uuid, src: sb->dmz_uuid);
1020	if (uuid_is_null(uuid: &sb_uuid)) {
1021	dmz_dev_err(dev, "NULL DM-Zoned uuid");
1022	return -ENXIO;
1023	} else if (uuid_is_null(uuid: &zmd->uuid)) {
1024	uuid_copy(dst: &zmd->uuid, src: &sb_uuid);
1025	} else if (!uuid_equal(u1: &zmd->uuid, u2: &sb_uuid)) {
1026	dmz_dev_err(dev, "mismatching DM-Zoned uuid, is %pUl expected %pUl",
1027	&sb_uuid, &zmd->uuid);
1028	return -ENXIO;
1029	}
1030	if (!strlen(zmd->label))
1031	memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);
1032	else if (memcmp(p: zmd->label, q: sb->dmz_label, BDEVNAME_SIZE)) {
1033	dmz_dev_err(dev, "mismatching DM-Zoned label, is %s expected %s",
1034	sb->dmz_label, zmd->label);
1035	return -ENXIO;
1036	}
1037	import_uuid(dst: &dev->uuid, src: sb->dev_uuid);
1038	if (uuid_is_null(uuid: &dev->uuid)) {
1039	dmz_dev_err(dev, "NULL device uuid");
1040	return -ENXIO;
1041	}
1042
1043	if (tertiary) {
1044	/*
1045	* Generation number should be 0, but it doesn't
1046	* really matter if it isn't.
1047	*/
1048	if (gen != 0)
1049	dmz_dev_warn(dev, "Invalid generation %llu",
1050	gen);
1051	return 0;
1052	}
1053	}
1054
1055	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)
1056	>> zmd->zone_nr_blocks_shift;
1057	if (!nr_meta_zones ||
1058	(zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||
1059	(zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {
1060	dmz_dev_err(dev, "Invalid number of metadata blocks");
1061	return -ENXIO;
1062	}
1063
1064	if (!le32_to_cpu(sb->nr_reserved_seq) ||
1065	le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {
1066	dmz_dev_err(dev, "Invalid number of reserved sequential zones");
1067	return -ENXIO;
1068	}
1069
1070	nr_data_zones = zmd->nr_useable_zones -
1071	(nr_meta_zones * 2 + le32_to_cpu(sb->nr_reserved_seq));
1072	if (le32_to_cpu(sb->nr_chunks) > nr_data_zones) {
1073	dmz_dev_err(dev, "Invalid number of chunks %u / %u",
1074	le32_to_cpu(sb->nr_chunks), nr_data_zones);
1075	return -ENXIO;
1076	}
1077
1078	/* OK */
1079	zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);
1080	zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);
1081	zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);
1082	zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);
1083	zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);
1084	zmd->nr_meta_zones = nr_meta_zones;
1085	zmd->nr_data_zones = nr_data_zones;
1086
1087	return 0;
1088	}
1089
1090	/*
1091	* Read the first or second super block from disk.
1092	*/
1093	static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
1094	{
1095	dmz_zmd_debug(zmd, "read superblock set %d dev %pg block %llu",
1096	set, sb->dev->bdev, sb->block);
1097
1098	return dmz_rdwr_block(dev: sb->dev, op: REQ_OP_READ,
1099	block: sb->block, page: sb->mblk->page);
1100	}
1101
1102	/*
1103	* Determine the position of the secondary super blocks on disk.
1104	* This is used only if a corruption of the primary super block
1105	* is detected.
1106	*/
1107	static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)
1108	{
1109	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
1110	struct dmz_mblock *mblk;
1111	unsigned int zone_id = zmd->sb[0].zone->id;
1112	int i;
1113
1114	/* Allocate a block */
1115	mblk = dmz_alloc_mblock(zmd, mblk_no: 0);
1116	if (!mblk)
1117	return -ENOMEM;
1118
1119	zmd->sb[1].mblk = mblk;
1120	zmd->sb[1].sb = mblk->data;
1121
1122	/* Bad first super block: search for the second one */
1123	zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;
1124	zmd->sb[1].zone = dmz_get(zmd, zone_id: zone_id + 1);
1125	zmd->sb[1].dev = zmd->sb[0].dev;
1126	for (i = 1; i < zmd->nr_rnd_zones; i++) {
1127	if (dmz_read_sb(zmd, sb: &zmd->sb[1], set: 1) != 0)
1128	break;
1129	if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)
1130	return 0;
1131	zmd->sb[1].block += zone_nr_blocks;
1132	zmd->sb[1].zone = dmz_get(zmd, zone_id: zone_id + i);
1133	}
1134
1135	dmz_free_mblock(zmd, mblk);
1136	zmd->sb[1].mblk = NULL;
1137	zmd->sb[1].zone = NULL;
1138	zmd->sb[1].dev = NULL;
1139
1140	return -EIO;
1141	}
1142
1143	/*
1144	* Read a super block from disk.
1145	*/
1146	static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
1147	{
1148	struct dmz_mblock *mblk;
1149	int ret;
1150
1151	/* Allocate a block */
1152	mblk = dmz_alloc_mblock(zmd, mblk_no: 0);
1153	if (!mblk)
1154	return -ENOMEM;
1155
1156	sb->mblk = mblk;
1157	sb->sb = mblk->data;
1158
1159	/* Read super block */
1160	ret = dmz_read_sb(zmd, sb, set);
1161	if (ret) {
1162	dmz_free_mblock(zmd, mblk);
1163	sb->mblk = NULL;
1164	return ret;
1165	}
1166
1167	return 0;
1168	}
1169
1170	/*
1171	* Recover a metadata set.
1172	*/
1173	static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)
1174	{
1175	unsigned int src_set = dst_set ^ 0x1;
1176	struct page *page;
1177	int i, ret;
1178
1179	dmz_dev_warn(zmd->sb[dst_set].dev,
1180	"Metadata set %u invalid: recovering", dst_set);
1181
1182	if (dst_set == 0)
1183	zmd->sb[0].block = dmz_start_block(zmd, zone: zmd->sb[0].zone);
1184	else
1185	zmd->sb[1].block = dmz_start_block(zmd, zone: zmd->sb[1].zone);
1186
1187	page = alloc_page(GFP_NOIO);
1188	if (!page)
1189	return -ENOMEM;
1190
1191	/* Copy metadata blocks */
1192	for (i = 1; i < zmd->nr_meta_blocks; i++) {
1193	ret = dmz_rdwr_block(dev: zmd->sb[src_set].dev, op: REQ_OP_READ,
1194	block: zmd->sb[src_set].block + i, page);
1195	if (ret)
1196	goto out;
1197	ret = dmz_rdwr_block(dev: zmd->sb[dst_set].dev, op: REQ_OP_WRITE,
1198	block: zmd->sb[dst_set].block + i, page);
1199	if (ret)
1200	goto out;
1201	}
1202
1203	/* Finalize with the super block */
1204	if (!zmd->sb[dst_set].mblk) {
1205	zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, mblk_no: 0);
1206	if (!zmd->sb[dst_set].mblk) {
1207	ret = -ENOMEM;
1208	goto out;
1209	}
1210	zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;
1211	}
1212
1213	ret = dmz_write_sb(zmd, set: dst_set);
1214	out:
1215	__free_pages(page, order: 0);
1216
1217	return ret;
1218	}
1219
1220	/*
1221	* Get super block from disk.
1222	*/
1223	static int dmz_load_sb(struct dmz_metadata *zmd)
1224	{
1225	bool sb_good[2] = {false, false};
1226	u64 sb_gen[2] = {0, 0};
1227	int ret;
1228
1229	if (!zmd->sb[0].zone) {
1230	dmz_zmd_err(zmd, "Primary super block zone not set");
1231	return -ENXIO;
1232	}
1233
1234	/* Read and check the primary super block */
1235	zmd->sb[0].block = dmz_start_block(zmd, zone: zmd->sb[0].zone);
1236	zmd->sb[0].dev = zmd->sb[0].zone->dev;
1237	ret = dmz_get_sb(zmd, sb: &zmd->sb[0], set: 0);
1238	if (ret) {
1239	dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");
1240	return ret;
1241	}
1242
1243	ret = dmz_check_sb(zmd, dsb: &zmd->sb[0], tertiary: false);
1244
1245	/* Read and check secondary super block */
1246	if (ret == 0) {
1247	sb_good[0] = true;
1248	if (!zmd->sb[1].zone) {
1249	unsigned int zone_id =
1250	zmd->sb[0].zone->id + zmd->nr_meta_zones;
1251
1252	zmd->sb[1].zone = dmz_get(zmd, zone_id);
1253	}
1254	zmd->sb[1].block = dmz_start_block(zmd, zone: zmd->sb[1].zone);
1255	zmd->sb[1].dev = zmd->sb[0].dev;
1256	ret = dmz_get_sb(zmd, sb: &zmd->sb[1], set: 1);
1257	} else
1258	ret = dmz_lookup_secondary_sb(zmd);
1259
1260	if (ret) {
1261	dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");
1262	return ret;
1263	}
1264
1265	ret = dmz_check_sb(zmd, dsb: &zmd->sb[1], tertiary: false);
1266	if (ret == 0)
1267	sb_good[1] = true;
1268
1269	/* Use highest generation sb first */
1270	if (!sb_good[0] && !sb_good[1]) {
1271	dmz_zmd_err(zmd, "No valid super block found");
1272	return -EIO;
1273	}
1274
1275	if (sb_good[0])
1276	sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);
1277	else {
1278	ret = dmz_recover_mblocks(zmd, dst_set: 0);
1279	if (ret) {
1280	dmz_dev_err(zmd->sb[0].dev,
1281	"Recovery of superblock 0 failed");
1282	return -EIO;
1283	}
1284	}
1285
1286	if (sb_good[1])
1287	sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);
1288	else {
1289	ret = dmz_recover_mblocks(zmd, dst_set: 1);
1290
1291	if (ret) {
1292	dmz_dev_err(zmd->sb[1].dev,
1293	"Recovery of superblock 1 failed");
1294	return -EIO;
1295	}
1296	}
1297
1298	if (sb_gen[0] >= sb_gen[1]) {
1299	zmd->sb_gen = sb_gen[0];
1300	zmd->mblk_primary = 0;
1301	} else {
1302	zmd->sb_gen = sb_gen[1];
1303	zmd->mblk_primary = 1;
1304	}
1305
1306	dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,
1307	"Using super block %u (gen %llu)",
1308	zmd->mblk_primary, zmd->sb_gen);
1309
1310	if (zmd->sb_version > 1) {
1311	int i;
1312	struct dmz_sb *sb;
1313
1314	sb = kzalloc(sizeof(struct dmz_sb), GFP_KERNEL);
1315	if (!sb)
1316	return -ENOMEM;
1317	for (i = 1; i < zmd->nr_devs; i++) {
1318	sb->block = 0;
1319	sb->zone = dmz_get(zmd, zone_id: zmd->dev[i].zone_offset);
1320	sb->dev = &zmd->dev[i];
1321	if (!dmz_is_meta(sb->zone)) {
1322	dmz_dev_err(sb->dev,
1323	"Tertiary super block zone %u not marked as metadata zone",
1324	sb->zone->id);
1325	ret = -EINVAL;
1326	goto out_kfree;
1327	}
1328	ret = dmz_get_sb(zmd, sb, set: i + 1);
1329	if (ret) {
1330	dmz_dev_err(sb->dev,
1331	"Read tertiary super block failed");
1332	dmz_free_mblock(zmd, mblk: sb->mblk);
1333	goto out_kfree;
1334	}
1335	ret = dmz_check_sb(zmd, dsb: sb, tertiary: true);
1336	dmz_free_mblock(zmd, mblk: sb->mblk);
1337	if (ret == -EINVAL)
1338	goto out_kfree;
1339	}
1340	out_kfree:
1341	kfree(objp: sb);
1342	}
1343	return ret;
1344	}
1345
1346	/*
1347	* Initialize a zone descriptor.
1348	*/
1349	static int dmz_init_zone(struct blk_zone *blkz, unsigned int num, void *data)
1350	{
1351	struct dmz_dev *dev = data;
1352	struct dmz_metadata *zmd = dev->metadata;
1353	int idx = num + dev->zone_offset;
1354	struct dm_zone *zone;
1355
1356	zone = dmz_insert(zmd, zone_id: idx, dev);
1357	if (IS_ERR(ptr: zone))
1358	return PTR_ERR(ptr: zone);
1359
1360	if (blkz->len != zmd->zone_nr_sectors) {
1361	if (zmd->sb_version > 1) {
1362	/* Ignore the eventual runt (smaller) zone */
1363	set_bit(nr: DMZ_OFFLINE, addr: &zone->flags);
1364	return 0;
1365	} else if (blkz->start + blkz->len == dev->capacity)
1366	return 0;
1367	return -ENXIO;
1368	}
1369
1370	/*
1371	* Devices that have zones with a capacity smaller than the zone size
1372	* (e.g. NVMe zoned namespaces) are not supported.
1373	*/
1374	if (blkz->capacity != blkz->len)
1375	return -ENXIO;
1376
1377	switch (blkz->type) {
1378	case BLK_ZONE_TYPE_CONVENTIONAL:
1379	set_bit(nr: DMZ_RND, addr: &zone->flags);
1380	break;
1381	case BLK_ZONE_TYPE_SEQWRITE_REQ:
1382	case BLK_ZONE_TYPE_SEQWRITE_PREF:
1383	set_bit(nr: DMZ_SEQ, addr: &zone->flags);
1384	break;
1385	default:
1386	return -ENXIO;
1387	}
1388
1389	if (dmz_is_rnd(zone))
1390	zone->wp_block = 0;
1391	else
1392	zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
1393
1394	if (blkz->cond == BLK_ZONE_COND_OFFLINE)
1395	set_bit(nr: DMZ_OFFLINE, addr: &zone->flags);
1396	else if (blkz->cond == BLK_ZONE_COND_READONLY)
1397	set_bit(nr: DMZ_READ_ONLY, addr: &zone->flags);
1398	else {
1399	zmd->nr_useable_zones++;
1400	if (dmz_is_rnd(zone)) {
1401	zmd->nr_rnd_zones++;
1402	if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {
1403	/* Primary super block zone */
1404	zmd->sb[0].zone = zone;
1405	}
1406	}
1407	if (zmd->nr_devs > 1 && num == 0) {
1408	/*
1409	* Tertiary superblock zones are always at the
1410	* start of the zoned devices, so mark them
1411	* as metadata zone.
1412	*/
1413	set_bit(nr: DMZ_META, addr: &zone->flags);
1414	}
1415	}
1416	return 0;
1417	}
1418
1419	static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)
1420	{
1421	int idx;
1422	sector_t zone_offset = 0;
1423
1424	for (idx = 0; idx < dev->nr_zones; idx++) {
1425	struct dm_zone *zone;
1426
1427	zone = dmz_insert(zmd, zone_id: idx, dev);
1428	if (IS_ERR(ptr: zone))
1429	return PTR_ERR(ptr: zone);
1430	set_bit(nr: DMZ_CACHE, addr: &zone->flags);
1431	zone->wp_block = 0;
1432	zmd->nr_cache_zones++;
1433	zmd->nr_useable_zones++;
1434	if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {
1435	/* Disable runt zone */
1436	set_bit(nr: DMZ_OFFLINE, addr: &zone->flags);
1437	break;
1438	}
1439	zone_offset += zmd->zone_nr_sectors;
1440	}
1441	return 0;
1442	}
1443
1444	/*
1445	* Free zones descriptors.
1446	*/
1447	static void dmz_drop_zones(struct dmz_metadata *zmd)
1448	{
1449	int idx;
1450
1451	for (idx = 0; idx < zmd->nr_zones; idx++) {
1452	struct dm_zone *zone = xa_load(&zmd->zones, index: idx);
1453
1454	kfree(objp: zone);
1455	xa_erase(&zmd->zones, index: idx);
1456	}
1457	xa_destroy(&zmd->zones);
1458	}
1459
1460	/*
1461	* Allocate and initialize zone descriptors using the zone
1462	* information from disk.
1463	*/
1464	static int dmz_init_zones(struct dmz_metadata *zmd)
1465	{
1466	int i, ret;
1467	struct dmz_dev *zoned_dev = &zmd->dev[0];
1468
1469	/* Init */
1470	zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;
1471	zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);
1472	zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);
1473	zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);
1474	zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;
1475	zmd->zone_nr_bitmap_blocks =
1476	max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);
1477	zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,
1478	DMZ_BLOCK_SIZE_BITS);
1479
1480	/* Allocate zone array */
1481	zmd->nr_zones = 0;
1482	for (i = 0; i < zmd->nr_devs; i++) {
1483	struct dmz_dev *dev = &zmd->dev[i];
1484
1485	dev->metadata = zmd;
1486	zmd->nr_zones += dev->nr_zones;
1487
1488	atomic_set(v: &dev->unmap_nr_rnd, i: 0);
1489	INIT_LIST_HEAD(list: &dev->unmap_rnd_list);
1490	INIT_LIST_HEAD(list: &dev->map_rnd_list);
1491
1492	atomic_set(v: &dev->unmap_nr_seq, i: 0);
1493	INIT_LIST_HEAD(list: &dev->unmap_seq_list);
1494	INIT_LIST_HEAD(list: &dev->map_seq_list);
1495	}
1496
1497	if (!zmd->nr_zones) {
1498	DMERR("(%s): No zones found", zmd->devname);
1499	return -ENXIO;
1500	}
1501	xa_init(xa: &zmd->zones);
1502
1503	DMDEBUG("(%s): Using %zu B for zone information",
1504	zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);
1505
1506	if (zmd->nr_devs > 1) {
1507	ret = dmz_emulate_zones(zmd, dev: &zmd->dev[0]);
1508	if (ret < 0) {
1509	DMDEBUG("(%s): Failed to emulate zones, error %d",
1510	zmd->devname, ret);
1511	dmz_drop_zones(zmd);
1512	return ret;
1513	}
1514
1515	/*
1516	* Primary superblock zone is always at zone 0 when multiple
1517	* drives are present.
1518	*/
1519	zmd->sb[0].zone = dmz_get(zmd, zone_id: 0);
1520
1521	for (i = 1; i < zmd->nr_devs; i++) {
1522	zoned_dev = &zmd->dev[i];
1523
1524	ret = blkdev_report_zones(bdev: zoned_dev->bdev, sector: 0,
1525	BLK_ALL_ZONES,
1526	cb: dmz_init_zone, data: zoned_dev);
1527	if (ret < 0) {
1528	DMDEBUG("(%s): Failed to report zones, error %d",
1529	zmd->devname, ret);
1530	dmz_drop_zones(zmd);
1531	return ret;
1532	}
1533	}
1534	return 0;
1535	}
1536
1537	/*
1538	* Get zone information and initialize zone descriptors. At the same
1539	* time, determine where the super block should be: first block of the
1540	* first randomly writable zone.
1541	*/
1542	ret = blkdev_report_zones(bdev: zoned_dev->bdev, sector: 0, BLK_ALL_ZONES,
1543	cb: dmz_init_zone, data: zoned_dev);
1544	if (ret < 0) {
1545	DMDEBUG("(%s): Failed to report zones, error %d",
1546	zmd->devname, ret);
1547	dmz_drop_zones(zmd);
1548	return ret;
1549	}
1550
1551	return 0;
1552	}
1553
1554	static int dmz_update_zone_cb(struct blk_zone *blkz, unsigned int idx,
1555	void *data)
1556	{
1557	struct dm_zone *zone = data;
1558
1559	clear_bit(nr: DMZ_OFFLINE, addr: &zone->flags);
1560	clear_bit(nr: DMZ_READ_ONLY, addr: &zone->flags);
1561	if (blkz->cond == BLK_ZONE_COND_OFFLINE)
1562	set_bit(nr: DMZ_OFFLINE, addr: &zone->flags);
1563	else if (blkz->cond == BLK_ZONE_COND_READONLY)
1564	set_bit(nr: DMZ_READ_ONLY, addr: &zone->flags);
1565
1566	if (dmz_is_seq(zone))
1567	zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
1568	else
1569	zone->wp_block = 0;
1570	return 0;
1571	}
1572
1573	/*
1574	* Update a zone information.
1575	*/
1576	static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1577	{
1578	struct dmz_dev *dev = zone->dev;
1579	unsigned int noio_flag;
1580	int ret;
1581
1582	if (dev->flags & DMZ_BDEV_REGULAR)
1583	return 0;
1584
1585	/*
1586	* Get zone information from disk. Since blkdev_report_zones() uses
1587	* GFP_KERNEL by default for memory allocations, set the per-task
1588	* PF_MEMALLOC_NOIO flag so that all allocations are done as if
1589	* GFP_NOIO was specified.
1590	*/
1591	noio_flag = memalloc_noio_save();
1592	ret = blkdev_report_zones(bdev: dev->bdev, sector: dmz_start_sect(zmd, zone), nr_zones: 1,
1593	cb: dmz_update_zone_cb, data: zone);
1594	memalloc_noio_restore(flags: noio_flag);
1595
1596	if (ret == 0)
1597	ret = -EIO;
1598	if (ret < 0) {
1599	dmz_dev_err(dev, "Get zone %u report failed",
1600	zone->id);
1601	dmz_check_bdev(dmz_dev: dev);
1602	return ret;
1603	}
1604
1605	return 0;
1606	}
1607
1608	/*
1609	* Check a zone write pointer position when the zone is marked
1610	* with the sequential write error flag.
1611	*/
1612	static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
1613	struct dm_zone *zone)
1614	{
1615	struct dmz_dev *dev = zone->dev;
1616	unsigned int wp = 0;
1617	int ret;
1618
1619	wp = zone->wp_block;
1620	ret = dmz_update_zone(zmd, zone);
1621	if (ret)
1622	return ret;
1623
1624	dmz_dev_warn(dev, "Processing zone %u write error (zone wp %u/%u)",
1625	zone->id, zone->wp_block, wp);
1626
1627	if (zone->wp_block < wp) {
1628	dmz_invalidate_blocks(zmd, zone, chunk_block: zone->wp_block,
1629	nr_blocks: wp - zone->wp_block);
1630	}
1631
1632	return 0;
1633	}
1634
1635	/*
1636	* Reset a zone write pointer.
1637	*/
1638	static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1639	{
1640	int ret;
1641
1642	/*
1643	* Ignore offline zones, read only zones,
1644	* and conventional zones.
1645	*/
1646	if (dmz_is_offline(zone) ||
1647	dmz_is_readonly(zone) ||
1648	dmz_is_rnd(zone))
1649	return 0;
1650
1651	if (!dmz_is_empty(zone) || dmz_seq_write_err(zone)) {
1652	struct dmz_dev *dev = zone->dev;
1653	unsigned int noio_flag;
1654
1655	noio_flag = memalloc_noio_save();
1656	ret = blkdev_zone_mgmt(bdev: dev->bdev, op: REQ_OP_ZONE_RESET,
1657	sectors: dmz_start_sect(zmd, zone),
1658	nr_sectors: zmd->zone_nr_sectors);
1659	memalloc_noio_restore(flags: noio_flag);
1660	if (ret) {
1661	dmz_dev_err(dev, "Reset zone %u failed %d",
1662	zone->id, ret);
1663	return ret;
1664	}
1665	}
1666
1667	/* Clear write error bit and rewind write pointer position */
1668	clear_bit(nr: DMZ_SEQ_WRITE_ERR, addr: &zone->flags);
1669	zone->wp_block = 0;
1670
1671	return 0;
1672	}
1673
1674	static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
1675
1676	/*
1677	* Initialize chunk mapping.
1678	*/
1679	static int dmz_load_mapping(struct dmz_metadata *zmd)
1680	{
1681	struct dm_zone *dzone, *bzone;
1682	struct dmz_mblock *dmap_mblk = NULL;
1683	struct dmz_map *dmap;
1684	unsigned int i = 0, e = 0, chunk = 0;
1685	unsigned int dzone_id;
1686	unsigned int bzone_id;
1687
1688	/* Metadata block array for the chunk mapping table */
1689	zmd->map_mblk = kcalloc(zmd->nr_map_blocks,
1690	sizeof(struct dmz_mblk *), GFP_KERNEL);
1691	if (!zmd->map_mblk)
1692	return -ENOMEM;
1693
1694	/* Get chunk mapping table blocks and initialize zone mapping */
1695	while (chunk < zmd->nr_chunks) {
1696	if (!dmap_mblk) {
1697	/* Get mapping block */
1698	dmap_mblk = dmz_get_mblock(zmd, mblk_no: i + 1);
1699	if (IS_ERR(ptr: dmap_mblk))
1700	return PTR_ERR(ptr: dmap_mblk);
1701	zmd->map_mblk[i] = dmap_mblk;
1702	dmap = dmap_mblk->data;
1703	i++;
1704	e = 0;
1705	}
1706
1707	/* Check data zone */
1708	dzone_id = le32_to_cpu(dmap[e].dzone_id);
1709	if (dzone_id == DMZ_MAP_UNMAPPED)
1710	goto next;
1711
1712	if (dzone_id >= zmd->nr_zones) {
1713	dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",
1714	chunk, dzone_id);
1715	return -EIO;
1716	}
1717
1718	dzone = dmz_get(zmd, zone_id: dzone_id);
1719	if (!dzone) {
1720	dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",
1721	chunk, dzone_id);
1722	return -EIO;
1723	}
1724	set_bit(nr: DMZ_DATA, addr: &dzone->flags);
1725	dzone->chunk = chunk;
1726	dmz_get_zone_weight(zmd, zone: dzone);
1727
1728	if (dmz_is_cache(dzone))
1729	list_add_tail(new: &dzone->link, head: &zmd->map_cache_list);
1730	else if (dmz_is_rnd(dzone))
1731	list_add_tail(new: &dzone->link, head: &dzone->dev->map_rnd_list);
1732	else
1733	list_add_tail(new: &dzone->link, head: &dzone->dev->map_seq_list);
1734
1735	/* Check buffer zone */
1736	bzone_id = le32_to_cpu(dmap[e].bzone_id);
1737	if (bzone_id == DMZ_MAP_UNMAPPED)
1738	goto next;
1739
1740	if (bzone_id >= zmd->nr_zones) {
1741	dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",
1742	chunk, bzone_id);
1743	return -EIO;
1744	}
1745
1746	bzone = dmz_get(zmd, zone_id: bzone_id);
1747	if (!bzone) {
1748	dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",
1749	chunk, bzone_id);
1750	return -EIO;
1751	}
1752	if (!dmz_is_rnd(bzone) && !dmz_is_cache(bzone)) {
1753	dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",
1754	chunk, bzone_id);
1755	return -EIO;
1756	}
1757
1758	set_bit(nr: DMZ_DATA, addr: &bzone->flags);
1759	set_bit(nr: DMZ_BUF, addr: &bzone->flags);
1760	bzone->chunk = chunk;
1761	bzone->bzone = dzone;
1762	dzone->bzone = bzone;
1763	dmz_get_zone_weight(zmd, zone: bzone);
1764	if (dmz_is_cache(bzone))
1765	list_add_tail(new: &bzone->link, head: &zmd->map_cache_list);
1766	else
1767	list_add_tail(new: &bzone->link, head: &bzone->dev->map_rnd_list);
1768	next:
1769	chunk++;
1770	e++;
1771	if (e >= DMZ_MAP_ENTRIES)
1772	dmap_mblk = NULL;
1773	}
1774
1775	/*
1776	* At this point, only meta zones and mapped data zones were
1777	* fully initialized. All remaining zones are unmapped data
1778	* zones. Finish initializing those here.
1779	*/
1780	for (i = 0; i < zmd->nr_zones; i++) {
1781	dzone = dmz_get(zmd, zone_id: i);
1782	if (!dzone)
1783	continue;
1784	if (dmz_is_meta(dzone))
1785	continue;
1786	if (dmz_is_offline(dzone))
1787	continue;
1788
1789	if (dmz_is_cache(dzone))
1790	zmd->nr_cache++;
1791	else if (dmz_is_rnd(dzone))
1792	dzone->dev->nr_rnd++;
1793	else
1794	dzone->dev->nr_seq++;
1795
1796	if (dmz_is_data(dzone)) {
1797	/* Already initialized */
1798	continue;
1799	}
1800
1801	/* Unmapped data zone */
1802	set_bit(nr: DMZ_DATA, addr: &dzone->flags);
1803	dzone->chunk = DMZ_MAP_UNMAPPED;
1804	if (dmz_is_cache(dzone)) {
1805	list_add_tail(new: &dzone->link, head: &zmd->unmap_cache_list);
1806	atomic_inc(v: &zmd->unmap_nr_cache);
1807	} else if (dmz_is_rnd(dzone)) {
1808	list_add_tail(new: &dzone->link,
1809	head: &dzone->dev->unmap_rnd_list);
1810	atomic_inc(v: &dzone->dev->unmap_nr_rnd);
1811	} else if (atomic_read(v: &zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {
1812	list_add_tail(new: &dzone->link, head: &zmd->reserved_seq_zones_list);
1813	set_bit(nr: DMZ_RESERVED, addr: &dzone->flags);
1814	atomic_inc(v: &zmd->nr_reserved_seq_zones);
1815	dzone->dev->nr_seq--;
1816	} else {
1817	list_add_tail(new: &dzone->link,
1818	head: &dzone->dev->unmap_seq_list);
1819	atomic_inc(v: &dzone->dev->unmap_nr_seq);
1820	}
1821	}
1822
1823	return 0;
1824	}
1825
1826	/*
1827	* Set a data chunk mapping.
1828	*/
1829	static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,
1830	unsigned int dzone_id, unsigned int bzone_id)
1831	{
1832	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
1833	struct dmz_map *dmap = dmap_mblk->data;
1834	int map_idx = chunk & DMZ_MAP_ENTRIES_MASK;
1835
1836	dmap[map_idx].dzone_id = cpu_to_le32(dzone_id);
1837	dmap[map_idx].bzone_id = cpu_to_le32(bzone_id);
1838	dmz_dirty_mblock(zmd, mblk: dmap_mblk);
1839	}
1840
1841	/*
1842	* The list of mapped zones is maintained in LRU order.
1843	* This rotates a zone at the end of its map list.
1844	*/
1845	static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1846	{
1847	if (list_empty(head: &zone->link))
1848	return;
1849
1850	list_del_init(entry: &zone->link);
1851	if (dmz_is_seq(zone)) {
1852	/* LRU rotate sequential zone */
1853	list_add_tail(new: &zone->link, head: &zone->dev->map_seq_list);
1854	} else if (dmz_is_cache(zone)) {
1855	/* LRU rotate cache zone */
1856	list_add_tail(new: &zone->link, head: &zmd->map_cache_list);
1857	} else {
1858	/* LRU rotate random zone */
1859	list_add_tail(new: &zone->link, head: &zone->dev->map_rnd_list);
1860	}
1861	}
1862
1863	/*
1864	* The list of mapped random zones is maintained
1865	* in LRU order. This rotates a zone at the end of the list.
1866	*/
1867	static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1868	{
1869	__dmz_lru_zone(zmd, zone);
1870	if (zone->bzone)
1871	__dmz_lru_zone(zmd, zone: zone->bzone);
1872	}
1873
1874	/*
1875	* Wait for any zone to be freed.
1876	*/
1877	static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)
1878	{
1879	DEFINE_WAIT(wait);
1880
1881	prepare_to_wait(wq_head: &zmd->free_wq, wq_entry: &wait, TASK_UNINTERRUPTIBLE);
1882	dmz_unlock_map(zmd);
1883	dmz_unlock_metadata(zmd);
1884
1885	io_schedule_timeout(HZ);
1886
1887	dmz_lock_metadata(zmd);
1888	dmz_lock_map(zmd);
1889	finish_wait(wq_head: &zmd->free_wq, wq_entry: &wait);
1890	}
1891
1892	/*
1893	* Lock a zone for reclaim (set the zone RECLAIM bit).
1894	* Returns false if the zone cannot be locked or if it is already locked
1895	* and 1 otherwise.
1896	*/
1897	int dmz_lock_zone_reclaim(struct dm_zone *zone)
1898	{
1899	/* Active zones cannot be reclaimed */
1900	if (dmz_is_active(zone))
1901	return 0;
1902
1903	return !test_and_set_bit(nr: DMZ_RECLAIM, addr: &zone->flags);
1904	}
1905
1906	/*
1907	* Clear a zone reclaim flag.
1908	*/
1909	void dmz_unlock_zone_reclaim(struct dm_zone *zone)
1910	{
1911	WARN_ON(dmz_is_active(zone));
1912	WARN_ON(!dmz_in_reclaim(zone));
1913
1914	clear_bit_unlock(nr: DMZ_RECLAIM, addr: &zone->flags);
1915	smp_mb__after_atomic();
1916	wake_up_bit(word: &zone->flags, bit: DMZ_RECLAIM);
1917	}
1918
1919	/*
1920	* Wait for a zone reclaim to complete.
1921	*/
1922	static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)
1923	{
1924	dmz_unlock_map(zmd);
1925	dmz_unlock_metadata(zmd);
1926	set_bit(nr: DMZ_RECLAIM_TERMINATE, addr: &zone->flags);
1927	wait_on_bit_timeout(word: &zone->flags, bit: DMZ_RECLAIM, TASK_UNINTERRUPTIBLE, HZ);
1928	clear_bit(nr: DMZ_RECLAIM_TERMINATE, addr: &zone->flags);
1929	dmz_lock_metadata(zmd);
1930	dmz_lock_map(zmd);
1931	}
1932
1933	/*
1934	* Select a cache or random write zone for reclaim.
1935	*/
1936	static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,
1937	unsigned int idx, bool idle)
1938	{
1939	struct dm_zone *dzone = NULL;
1940	struct dm_zone *zone, *maxw_z = NULL;
1941	struct list_head *zone_list;
1942
1943	/* If we have cache zones select from the cache zone list */
1944	if (zmd->nr_cache) {
1945	zone_list = &zmd->map_cache_list;
1946	/* Try to relaim random zones, too, when idle */
1947	if (idle && list_empty(head: zone_list))
1948	zone_list = &zmd->dev[idx].map_rnd_list;
1949	} else
1950	zone_list = &zmd->dev[idx].map_rnd_list;
1951
1952	/*
1953	* Find the buffer zone with the heaviest weight or the first (oldest)
1954	* data zone that can be reclaimed.
1955	*/
1956	list_for_each_entry(zone, zone_list, link) {
1957	if (dmz_is_buf(zone)) {
1958	dzone = zone->bzone;
1959	if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
1960	continue;
1961	if (!maxw_z || maxw_z->weight < dzone->weight)
1962	maxw_z = dzone;
1963	} else {
1964	dzone = zone;
1965	if (dmz_lock_zone_reclaim(zone: dzone))
1966	return dzone;
1967	}
1968	}
1969
1970	if (maxw_z && dmz_lock_zone_reclaim(zone: maxw_z))
1971	return maxw_z;
1972
1973	/*
1974	* If we come here, none of the zones inspected could be locked for
1975	* reclaim. Try again, being more aggressive, that is, find the
1976	* first zone that can be reclaimed regardless of its weitght.
1977	*/
1978	list_for_each_entry(zone, zone_list, link) {
1979	if (dmz_is_buf(zone)) {
1980	dzone = zone->bzone;
1981	if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
1982	continue;
1983	} else
1984	dzone = zone;
1985	if (dmz_lock_zone_reclaim(zone: dzone))
1986	return dzone;
1987	}
1988
1989	return NULL;
1990	}
1991
1992	/*
1993	* Select a buffered sequential zone for reclaim.
1994	*/
1995	static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,
1996	unsigned int idx)
1997	{
1998	struct dm_zone *zone;
1999
2000	list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {
2001	if (!zone->bzone)
2002	continue;
2003	if (dmz_lock_zone_reclaim(zone))
2004	return zone;
2005	}
2006
2007	return NULL;
2008	}
2009
2010	/*
2011	* Select a zone for reclaim.
2012	*/
2013	struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,
2014	unsigned int dev_idx, bool idle)
2015	{
2016	struct dm_zone *zone = NULL;
2017
2018	/*
2019	* Search for a zone candidate to reclaim: 2 cases are possible.
2020	* (1) There is no free sequential zones. Then a random data zone
2021	* cannot be reclaimed. So choose a sequential zone to reclaim so
2022	* that afterward a random zone can be reclaimed.
2023	* (2) At least one free sequential zone is available, then choose
2024	* the oldest random zone (data or buffer) that can be locked.
2025	*/
2026	dmz_lock_map(zmd);
2027	if (list_empty(head: &zmd->reserved_seq_zones_list))
2028	zone = dmz_get_seq_zone_for_reclaim(zmd, idx: dev_idx);
2029	if (!zone)
2030	zone = dmz_get_rnd_zone_for_reclaim(zmd, idx: dev_idx, idle);
2031	dmz_unlock_map(zmd);
2032
2033	return zone;
2034	}
2035
2036	/*
2037	* Get the zone mapping a chunk, if the chunk is mapped already.
2038	* If no mapping exist and the operation is WRITE, a zone is
2039	* allocated and used to map the chunk.
2040	* The zone returned will be set to the active state.
2041	*/
2042	struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,
2043	unsigned int chunk, enum req_op op)
2044	{
2045	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
2046	struct dmz_map *dmap = dmap_mblk->data;
2047	int dmap_idx = chunk & DMZ_MAP_ENTRIES_MASK;
2048	unsigned int dzone_id;
2049	struct dm_zone *dzone = NULL;
2050	int ret = 0;
2051	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
2052
2053	dmz_lock_map(zmd);
2054	again:
2055	/* Get the chunk mapping */
2056	dzone_id = le32_to_cpu(dmap[dmap_idx].dzone_id);
2057	if (dzone_id == DMZ_MAP_UNMAPPED) {
2058	/*
2059	* Read or discard in unmapped chunks are fine. But for
2060	* writes, we need a mapping, so get one.
2061	*/
2062	if (op != REQ_OP_WRITE)
2063	goto out;
2064
2065	/* Allocate a random zone */
2066	dzone = dmz_alloc_zone(zmd, dev_idx: 0, flags: alloc_flags);
2067	if (!dzone) {
2068	if (dmz_dev_is_dying(zmd)) {
2069	dzone = ERR_PTR(error: -EIO);
2070	goto out;
2071	}
2072	dmz_wait_for_free_zones(zmd);
2073	goto again;
2074	}
2075
2076	dmz_map_zone(zmd, zone: dzone, chunk);
2077
2078	} else {
2079	/* The chunk is already mapped: get the mapping zone */
2080	dzone = dmz_get(zmd, zone_id: dzone_id);
2081	if (!dzone) {
2082	dzone = ERR_PTR(error: -EIO);
2083	goto out;
2084	}
2085	if (dzone->chunk != chunk) {
2086	dzone = ERR_PTR(error: -EIO);
2087	goto out;
2088	}
2089
2090	/* Repair write pointer if the sequential dzone has error */
2091	if (dmz_seq_write_err(dzone)) {
2092	ret = dmz_handle_seq_write_err(zmd, zone: dzone);
2093	if (ret) {
2094	dzone = ERR_PTR(error: -EIO);
2095	goto out;
2096	}
2097	clear_bit(nr: DMZ_SEQ_WRITE_ERR, addr: &dzone->flags);
2098	}
2099	}
2100
2101	/*
2102	* If the zone is being reclaimed, the chunk mapping may change
2103	* to a different zone. So wait for reclaim and retry. Otherwise,
2104	* activate the zone (this will prevent reclaim from touching it).
2105	*/
2106	if (dmz_in_reclaim(dzone)) {
2107	dmz_wait_for_reclaim(zmd, zone: dzone);
2108	goto again;
2109	}
2110	dmz_activate_zone(zone: dzone);
2111	dmz_lru_zone(zmd, zone: dzone);
2112	out:
2113	dmz_unlock_map(zmd);
2114
2115	return dzone;
2116	}
2117
2118	/*
2119	* Write and discard change the block validity of data zones and their buffer
2120	* zones. Check here that valid blocks are still present. If all blocks are
2121	* invalid, the zones can be unmapped on the fly without waiting for reclaim
2122	* to do it.
2123	*/
2124	void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)
2125	{
2126	struct dm_zone *bzone;
2127
2128	dmz_lock_map(zmd);
2129
2130	bzone = dzone->bzone;
2131	if (bzone) {
2132	if (dmz_weight(bzone))
2133	dmz_lru_zone(zmd, zone: bzone);
2134	else {
2135	/* Empty buffer zone: reclaim it */
2136	dmz_unmap_zone(zmd, zone: bzone);
2137	dmz_free_zone(zmd, zone: bzone);
2138	bzone = NULL;
2139	}
2140	}
2141
2142	/* Deactivate the data zone */
2143	dmz_deactivate_zone(zone: dzone);
2144	if (dmz_is_active(zone: dzone) || bzone || dmz_weight(dzone))
2145	dmz_lru_zone(zmd, zone: dzone);
2146	else {
2147	/* Unbuffered inactive empty data zone: reclaim it */
2148	dmz_unmap_zone(zmd, zone: dzone);
2149	dmz_free_zone(zmd, zone: dzone);
2150	}
2151
2152	dmz_unlock_map(zmd);
2153	}
2154
2155	/*
2156	* Allocate and map a random zone to buffer a chunk
2157	* already mapped to a sequential zone.
2158	*/
2159	struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,
2160	struct dm_zone *dzone)
2161	{
2162	struct dm_zone *bzone;
2163	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
2164
2165	dmz_lock_map(zmd);
2166	again:
2167	bzone = dzone->bzone;
2168	if (bzone)
2169	goto out;
2170
2171	/* Allocate a random zone */
2172	bzone = dmz_alloc_zone(zmd, dev_idx: 0, flags: alloc_flags);
2173	if (!bzone) {
2174	if (dmz_dev_is_dying(zmd)) {
2175	bzone = ERR_PTR(error: -EIO);
2176	goto out;
2177	}
2178	dmz_wait_for_free_zones(zmd);
2179	goto again;
2180	}
2181
2182	/* Update the chunk mapping */
2183	dmz_set_chunk_mapping(zmd, chunk: dzone->chunk, dzone_id: dzone->id, bzone_id: bzone->id);
2184
2185	set_bit(nr: DMZ_BUF, addr: &bzone->flags);
2186	bzone->chunk = dzone->chunk;
2187	bzone->bzone = dzone;
2188	dzone->bzone = bzone;
2189	if (dmz_is_cache(bzone))
2190	list_add_tail(new: &bzone->link, head: &zmd->map_cache_list);
2191	else
2192	list_add_tail(new: &bzone->link, head: &bzone->dev->map_rnd_list);
2193	out:
2194	dmz_unlock_map(zmd);
2195
2196	return bzone;
2197	}
2198
2199	/*
2200	* Get an unmapped (free) zone.
2201	* This must be called with the mapping lock held.
2202	*/
2203	struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,
2204	unsigned long flags)
2205	{
2206	struct list_head *list;
2207	struct dm_zone *zone;
2208	int i;
2209
2210	/* Schedule reclaim to ensure free zones are available */
2211	if (!(flags & DMZ_ALLOC_RECLAIM)) {
2212	for (i = 0; i < zmd->nr_devs; i++)
2213	dmz_schedule_reclaim(zrc: zmd->dev[i].reclaim);
2214	}
2215
2216	i = 0;
2217	again:
2218	if (flags & DMZ_ALLOC_CACHE)
2219	list = &zmd->unmap_cache_list;
2220	else if (flags & DMZ_ALLOC_RND)
2221	list = &zmd->dev[dev_idx].unmap_rnd_list;
2222	else
2223	list = &zmd->dev[dev_idx].unmap_seq_list;
2224
2225	if (list_empty(head: list)) {
2226	/*
2227	* No free zone: return NULL if this is for not reclaim.
2228	*/
2229	if (!(flags & DMZ_ALLOC_RECLAIM))
2230	return NULL;
2231	/*
2232	* Try to allocate from other devices
2233	*/
2234	if (i < zmd->nr_devs) {
2235	dev_idx = (dev_idx + 1) % zmd->nr_devs;
2236	i++;
2237	goto again;
2238	}
2239
2240	/*
2241	* Fallback to the reserved sequential zones
2242	*/
2243	zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,
2244	struct dm_zone, link);
2245	if (zone) {
2246	list_del_init(entry: &zone->link);
2247	atomic_dec(v: &zmd->nr_reserved_seq_zones);
2248	}
2249	return zone;
2250	}
2251
2252	zone = list_first_entry(list, struct dm_zone, link);
2253	list_del_init(entry: &zone->link);
2254
2255	if (dmz_is_cache(zone))
2256	atomic_dec(v: &zmd->unmap_nr_cache);
2257	else if (dmz_is_rnd(zone))
2258	atomic_dec(v: &zone->dev->unmap_nr_rnd);
2259	else
2260	atomic_dec(v: &zone->dev->unmap_nr_seq);
2261
2262	if (dmz_is_offline(zone)) {
2263	dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);
2264	zone = NULL;
2265	goto again;
2266	}
2267	if (dmz_is_meta(zone)) {
2268	dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);
2269	zone = NULL;
2270	goto again;
2271	}
2272	return zone;
2273	}
2274
2275	/*
2276	* Free a zone.
2277	* This must be called with the mapping lock held.
2278	*/
2279	void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
2280	{
2281	/* If this is a sequential zone, reset it */
2282	if (dmz_is_seq(zone))
2283	dmz_reset_zone(zmd, zone);
2284
2285	/* Return the zone to its type unmap list */
2286	if (dmz_is_cache(zone)) {
2287	list_add_tail(new: &zone->link, head: &zmd->unmap_cache_list);
2288	atomic_inc(v: &zmd->unmap_nr_cache);
2289	} else if (dmz_is_rnd(zone)) {
2290	list_add_tail(new: &zone->link, head: &zone->dev->unmap_rnd_list);
2291	atomic_inc(v: &zone->dev->unmap_nr_rnd);
2292	} else if (dmz_is_reserved(zone)) {
2293	list_add_tail(new: &zone->link, head: &zmd->reserved_seq_zones_list);
2294	atomic_inc(v: &zmd->nr_reserved_seq_zones);
2295	} else {
2296	list_add_tail(new: &zone->link, head: &zone->dev->unmap_seq_list);
2297	atomic_inc(v: &zone->dev->unmap_nr_seq);
2298	}
2299
2300	wake_up_all(&zmd->free_wq);
2301	}
2302
2303	/*
2304	* Map a chunk to a zone.
2305	* This must be called with the mapping lock held.
2306	*/
2307	void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,
2308	unsigned int chunk)
2309	{
2310	/* Set the chunk mapping */
2311	dmz_set_chunk_mapping(zmd, chunk, dzone_id: dzone->id,
2312	DMZ_MAP_UNMAPPED);
2313	dzone->chunk = chunk;
2314	if (dmz_is_cache(dzone))
2315	list_add_tail(new: &dzone->link, head: &zmd->map_cache_list);
2316	else if (dmz_is_rnd(dzone))
2317	list_add_tail(new: &dzone->link, head: &dzone->dev->map_rnd_list);
2318	else
2319	list_add_tail(new: &dzone->link, head: &dzone->dev->map_seq_list);
2320	}
2321
2322	/*
2323	* Unmap a zone.
2324	* This must be called with the mapping lock held.
2325	*/
2326	void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
2327	{
2328	unsigned int chunk = zone->chunk;
2329	unsigned int dzone_id;
2330
2331	if (chunk == DMZ_MAP_UNMAPPED) {
2332	/* Already unmapped */
2333	return;
2334	}
2335
2336	if (test_and_clear_bit(nr: DMZ_BUF, addr: &zone->flags)) {
2337	/*
2338	* Unmapping the chunk buffer zone: clear only
2339	* the chunk buffer mapping
2340	*/
2341	dzone_id = zone->bzone->id;
2342	zone->bzone->bzone = NULL;
2343	zone->bzone = NULL;
2344
2345	} else {
2346	/*
2347	* Unmapping the chunk data zone: the zone must
2348	* not be buffered.
2349	*/
2350	if (WARN_ON(zone->bzone)) {
2351	zone->bzone->bzone = NULL;
2352	zone->bzone = NULL;
2353	}
2354	dzone_id = DMZ_MAP_UNMAPPED;
2355	}
2356
2357	dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);
2358
2359	zone->chunk = DMZ_MAP_UNMAPPED;
2360	list_del_init(entry: &zone->link);
2361	}
2362
2363	/*
2364	* Set @nr_bits bits in @bitmap starting from @bit.
2365	* Return the number of bits changed from 0 to 1.
2366	*/
2367	static unsigned int dmz_set_bits(unsigned long *bitmap,
2368	unsigned int bit, unsigned int nr_bits)
2369	{
2370	unsigned long *addr;
2371	unsigned int end = bit + nr_bits;
2372	unsigned int n = 0;
2373
2374	while (bit < end) {
2375	if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2376	((end - bit) >= BITS_PER_LONG)) {
2377	/* Try to set the whole word at once */
2378	addr = bitmap + BIT_WORD(bit);
2379	if (*addr == 0) {
2380	*addr = ULONG_MAX;
2381	n += BITS_PER_LONG;
2382	bit += BITS_PER_LONG;
2383	continue;
2384	}
2385	}
2386
2387	if (!test_and_set_bit(nr: bit, addr: bitmap))
2388	n++;
2389	bit++;
2390	}
2391
2392	return n;
2393	}
2394
2395	/*
2396	* Get the bitmap block storing the bit for chunk_block in zone.
2397	*/
2398	static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,
2399	struct dm_zone *zone,
2400	sector_t chunk_block)
2401	{
2402	sector_t bitmap_block = 1 + zmd->nr_map_blocks +
2403	(sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +
2404	(chunk_block >> DMZ_BLOCK_SHIFT_BITS);
2405
2406	return dmz_get_mblock(zmd, mblk_no: bitmap_block);
2407	}
2408
2409	/*
2410	* Copy the valid blocks bitmap of from_zone to the bitmap of to_zone.
2411	*/
2412	int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
2413	struct dm_zone *to_zone)
2414	{
2415	struct dmz_mblock *from_mblk, *to_mblk;
2416	sector_t chunk_block = 0;
2417
2418	/* Get the zones bitmap blocks */
2419	while (chunk_block < zmd->zone_nr_blocks) {
2420	from_mblk = dmz_get_bitmap(zmd, zone: from_zone, chunk_block);
2421	if (IS_ERR(ptr: from_mblk))
2422	return PTR_ERR(ptr: from_mblk);
2423	to_mblk = dmz_get_bitmap(zmd, zone: to_zone, chunk_block);
2424	if (IS_ERR(ptr: to_mblk)) {
2425	dmz_release_mblock(zmd, mblk: from_mblk);
2426	return PTR_ERR(ptr: to_mblk);
2427	}
2428
2429	memcpy(to_mblk->data, from_mblk->data, DMZ_BLOCK_SIZE);
2430	dmz_dirty_mblock(zmd, mblk: to_mblk);
2431
2432	dmz_release_mblock(zmd, mblk: to_mblk);
2433	dmz_release_mblock(zmd, mblk: from_mblk);
2434
2435	chunk_block += zmd->zone_bits_per_mblk;
2436	}
2437
2438	to_zone->weight = from_zone->weight;
2439
2440	return 0;
2441	}
2442
2443	/*
2444	* Merge the valid blocks bitmap of from_zone into the bitmap of to_zone,
2445	* starting from chunk_block.
2446	*/
2447	int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
2448	struct dm_zone *to_zone, sector_t chunk_block)
2449	{
2450	unsigned int nr_blocks;
2451	int ret;
2452
2453	/* Get the zones bitmap blocks */
2454	while (chunk_block < zmd->zone_nr_blocks) {
2455	/* Get a valid region from the source zone */
2456	ret = dmz_first_valid_block(zmd, zone: from_zone, chunk_block: &chunk_block);
2457	if (ret <= 0)
2458	return ret;
2459
2460	nr_blocks = ret;
2461	ret = dmz_validate_blocks(zmd, zone: to_zone, chunk_block, nr_blocks);
2462	if (ret)
2463	return ret;
2464
2465	chunk_block += nr_blocks;
2466	}
2467
2468	return 0;
2469	}
2470
2471	/*
2472	* Validate all the blocks in the range [block..block+nr_blocks-1].
2473	*/
2474	int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
2475	sector_t chunk_block, unsigned int nr_blocks)
2476	{
2477	unsigned int count, bit, nr_bits;
2478	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
2479	struct dmz_mblock *mblk;
2480	unsigned int n = 0;
2481
2482	dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",
2483	zone->id, (unsigned long long)chunk_block,
2484	nr_blocks);
2485
2486	WARN_ON(chunk_block + nr_blocks > zone_nr_blocks);
2487
2488	while (nr_blocks) {
2489	/* Get bitmap block */
2490	mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2491	if (IS_ERR(ptr: mblk))
2492	return PTR_ERR(ptr: mblk);
2493
2494	/* Set bits */
2495	bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2496	nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2497
2498	count = dmz_set_bits(bitmap: (unsigned long *)mblk->data, bit, nr_bits);
2499	if (count) {
2500	dmz_dirty_mblock(zmd, mblk);
2501	n += count;
2502	}
2503	dmz_release_mblock(zmd, mblk);
2504
2505	nr_blocks -= nr_bits;
2506	chunk_block += nr_bits;
2507	}
2508
2509	if (likely(zone->weight + n <= zone_nr_blocks))
2510	zone->weight += n;
2511	else {
2512	dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",
2513	zone->id, zone->weight,
2514	zone_nr_blocks - n);
2515	zone->weight = zone_nr_blocks;
2516	}
2517
2518	return 0;
2519	}
2520
2521	/*
2522	* Clear nr_bits bits in bitmap starting from bit.
2523	* Return the number of bits cleared.
2524	*/
2525	static int dmz_clear_bits(unsigned long *bitmap, int bit, int nr_bits)
2526	{
2527	unsigned long *addr;
2528	int end = bit + nr_bits;
2529	int n = 0;
2530
2531	while (bit < end) {
2532	if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2533	((end - bit) >= BITS_PER_LONG)) {
2534	/* Try to clear whole word at once */
2535	addr = bitmap + BIT_WORD(bit);
2536	if (*addr == ULONG_MAX) {
2537	*addr = 0;
2538	n += BITS_PER_LONG;
2539	bit += BITS_PER_LONG;
2540	continue;
2541	}
2542	}
2543
2544	if (test_and_clear_bit(nr: bit, addr: bitmap))
2545	n++;
2546	bit++;
2547	}
2548
2549	return n;
2550	}
2551
2552	/*
2553	* Invalidate all the blocks in the range [block..block+nr_blocks-1].
2554	*/
2555	int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
2556	sector_t chunk_block, unsigned int nr_blocks)
2557	{
2558	unsigned int count, bit, nr_bits;
2559	struct dmz_mblock *mblk;
2560	unsigned int n = 0;
2561
2562	dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",
2563	zone->id, (u64)chunk_block, nr_blocks);
2564
2565	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
2566
2567	while (nr_blocks) {
2568	/* Get bitmap block */
2569	mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2570	if (IS_ERR(ptr: mblk))
2571	return PTR_ERR(ptr: mblk);
2572
2573	/* Clear bits */
2574	bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2575	nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2576
2577	count = dmz_clear_bits(bitmap: (unsigned long *)mblk->data,
2578	bit, nr_bits);
2579	if (count) {
2580	dmz_dirty_mblock(zmd, mblk);
2581	n += count;
2582	}
2583	dmz_release_mblock(zmd, mblk);
2584
2585	nr_blocks -= nr_bits;
2586	chunk_block += nr_bits;
2587	}
2588
2589	if (zone->weight >= n)
2590	zone->weight -= n;
2591	else {
2592	dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",
2593	zone->id, zone->weight, n);
2594	zone->weight = 0;
2595	}
2596
2597	return 0;
2598	}
2599
2600	/*
2601	* Get a block bit value.
2602	*/
2603	static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2604	sector_t chunk_block)
2605	{
2606	struct dmz_mblock *mblk;
2607	int ret;
2608
2609	WARN_ON(chunk_block >= zmd->zone_nr_blocks);
2610
2611	/* Get bitmap block */
2612	mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2613	if (IS_ERR(ptr: mblk))
2614	return PTR_ERR(ptr: mblk);
2615
2616	/* Get offset */
2617	ret = test_bit(chunk_block & DMZ_BLOCK_MASK_BITS,
2618	(unsigned long *) mblk->data) != 0;
2619
2620	dmz_release_mblock(zmd, mblk);
2621
2622	return ret;
2623	}
2624
2625	/*
2626	* Return the number of blocks from chunk_block to the first block with a bit
2627	* value specified by set. Search at most nr_blocks blocks from chunk_block.
2628	*/
2629	static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2630	sector_t chunk_block, unsigned int nr_blocks,
2631	int set)
2632	{
2633	struct dmz_mblock *mblk;
2634	unsigned int bit, set_bit, nr_bits;
2635	unsigned int zone_bits = zmd->zone_bits_per_mblk;
2636	unsigned long *bitmap;
2637	int n = 0;
2638
2639	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
2640
2641	while (nr_blocks) {
2642	/* Get bitmap block */
2643	mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2644	if (IS_ERR(ptr: mblk))
2645	return PTR_ERR(ptr: mblk);
2646
2647	/* Get offset */
2648	bitmap = (unsigned long *) mblk->data;
2649	bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2650	nr_bits = min(nr_blocks, zone_bits - bit);
2651	if (set)
2652	set_bit = find_next_bit(addr: bitmap, size: zone_bits, offset: bit);
2653	else
2654	set_bit = find_next_zero_bit(addr: bitmap, size: zone_bits, offset: bit);
2655	dmz_release_mblock(zmd, mblk);
2656
2657	n += set_bit - bit;
2658	if (set_bit < zone_bits)
2659	break;
2660
2661	nr_blocks -= nr_bits;
2662	chunk_block += nr_bits;
2663	}
2664
2665	return n;
2666	}
2667
2668	/*
2669	* Test if chunk_block is valid. If it is, the number of consecutive
2670	* valid blocks from chunk_block will be returned.
2671	*/
2672	int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,
2673	sector_t chunk_block)
2674	{
2675	int valid;
2676
2677	valid = dmz_test_block(zmd, zone, chunk_block);
2678	if (valid <= 0)
2679	return valid;
2680
2681	/* The block is valid: get the number of valid blocks from block */
2682	return dmz_to_next_set_block(zmd, zone, chunk_block,
2683	nr_blocks: zmd->zone_nr_blocks - chunk_block, set: 0);
2684	}
2685
2686	/*
2687	* Find the first valid block from @chunk_block in @zone.
2688	* If such a block is found, its number is returned using
2689	* @chunk_block and the total number of valid blocks from @chunk_block
2690	* is returned.
2691	*/
2692	int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2693	sector_t *chunk_block)
2694	{
2695	sector_t start_block = *chunk_block;
2696	int ret;
2697
2698	ret = dmz_to_next_set_block(zmd, zone, chunk_block: start_block,
2699	nr_blocks: zmd->zone_nr_blocks - start_block, set: 1);
2700	if (ret < 0)
2701	return ret;
2702
2703	start_block += ret;
2704	*chunk_block = start_block;
2705
2706	return dmz_to_next_set_block(zmd, zone, chunk_block: start_block,
2707	nr_blocks: zmd->zone_nr_blocks - start_block, set: 0);
2708	}
2709
2710	/*
2711	* Count the number of bits set starting from bit up to bit + nr_bits - 1.
2712	*/
2713	static int dmz_count_bits(void *bitmap, int bit, int nr_bits)
2714	{
2715	unsigned long *addr;
2716	int end = bit + nr_bits;
2717	int n = 0;
2718
2719	while (bit < end) {
2720	if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2721	((end - bit) >= BITS_PER_LONG)) {
2722	addr = (unsigned long *)bitmap + BIT_WORD(bit);
2723	if (*addr == ULONG_MAX) {
2724	n += BITS_PER_LONG;
2725	bit += BITS_PER_LONG;
2726	continue;
2727	}
2728	}
2729
2730	if (test_bit(bit, bitmap))
2731	n++;
2732	bit++;
2733	}
2734
2735	return n;
2736	}
2737
2738	/*
2739	* Get a zone weight.
2740	*/
2741	static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)
2742	{
2743	struct dmz_mblock *mblk;
2744	sector_t chunk_block = 0;
2745	unsigned int bit, nr_bits;
2746	unsigned int nr_blocks = zmd->zone_nr_blocks;
2747	void *bitmap;
2748	int n = 0;
2749
2750	while (nr_blocks) {
2751	/* Get bitmap block */
2752	mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2753	if (IS_ERR(ptr: mblk)) {
2754	n = 0;
2755	break;
2756	}
2757
2758	/* Count bits in this block */
2759	bitmap = mblk->data;
2760	bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2761	nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2762	n += dmz_count_bits(bitmap, bit, nr_bits);
2763
2764	dmz_release_mblock(zmd, mblk);
2765
2766	nr_blocks -= nr_bits;
2767	chunk_block += nr_bits;
2768	}
2769
2770	zone->weight = n;
2771	}
2772
2773	/*
2774	* Cleanup the zoned metadata resources.
2775	*/
2776	static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
2777	{
2778	struct rb_root *root;
2779	struct dmz_mblock *mblk, *next;
2780	int i;
2781
2782	/* Release zone mapping resources */
2783	if (zmd->map_mblk) {
2784	for (i = 0; i < zmd->nr_map_blocks; i++)
2785	dmz_release_mblock(zmd, mblk: zmd->map_mblk[i]);
2786	kfree(objp: zmd->map_mblk);
2787	zmd->map_mblk = NULL;
2788	}
2789
2790	/* Release super blocks */
2791	for (i = 0; i < 2; i++) {
2792	if (zmd->sb[i].mblk) {
2793	dmz_free_mblock(zmd, mblk: zmd->sb[i].mblk);
2794	zmd->sb[i].mblk = NULL;
2795	}
2796	}
2797
2798	/* Free cached blocks */
2799	while (!list_empty(head: &zmd->mblk_dirty_list)) {
2800	mblk = list_first_entry(&zmd->mblk_dirty_list,
2801	struct dmz_mblock, link);
2802	dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",
2803	(u64)mblk->no, mblk->ref);
2804	list_del_init(entry: &mblk->link);
2805	rb_erase(&mblk->node, &zmd->mblk_rbtree);
2806	dmz_free_mblock(zmd, mblk);
2807	}
2808
2809	while (!list_empty(head: &zmd->mblk_lru_list)) {
2810	mblk = list_first_entry(&zmd->mblk_lru_list,
2811	struct dmz_mblock, link);
2812	list_del_init(entry: &mblk->link);
2813	rb_erase(&mblk->node, &zmd->mblk_rbtree);
2814	dmz_free_mblock(zmd, mblk);
2815	}
2816
2817	/* Sanity checks: the mblock rbtree should now be empty */
2818	root = &zmd->mblk_rbtree;
2819	rbtree_postorder_for_each_entry_safe(mblk, next, root, node) {
2820	dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",
2821	(u64)mblk->no, mblk->ref);
2822	mblk->ref = 0;
2823	dmz_free_mblock(zmd, mblk);
2824	}
2825
2826	/* Free the zone descriptors */
2827	dmz_drop_zones(zmd);
2828
2829	mutex_destroy(lock: &zmd->mblk_flush_lock);
2830	mutex_destroy(lock: &zmd->map_lock);
2831	}
2832
2833	static void dmz_print_dev(struct dmz_metadata *zmd, int num)
2834	{
2835	struct dmz_dev *dev = &zmd->dev[num];
2836
2837	if (!bdev_is_zoned(bdev: dev->bdev))
2838	dmz_dev_info(dev, "Regular block device");
2839	else
2840	dmz_dev_info(dev, "Host-managed zoned block device");
2841
2842	if (zmd->sb_version > 1) {
2843	sector_t sector_offset =
2844	dev->zone_offset << zmd->zone_nr_sectors_shift;
2845
2846	dmz_dev_info(dev, " %llu 512-byte logical sectors (offset %llu)",
2847	(u64)dev->capacity, (u64)sector_offset);
2848	dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors (offset %llu)",
2849	dev->nr_zones, (u64)zmd->zone_nr_sectors,
2850	(u64)dev->zone_offset);
2851	} else {
2852	dmz_dev_info(dev, " %llu 512-byte logical sectors",
2853	(u64)dev->capacity);
2854	dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors",
2855	dev->nr_zones, (u64)zmd->zone_nr_sectors);
2856	}
2857	}
2858
2859	/*
2860	* Initialize the zoned metadata.
2861	*/
2862	int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev,
2863	struct dmz_metadata **metadata,
2864	const char *devname)
2865	{
2866	struct dmz_metadata *zmd;
2867	unsigned int i;
2868	struct dm_zone *zone;
2869	int ret;
2870
2871	zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);
2872	if (!zmd)
2873	return -ENOMEM;
2874
2875	strcpy(p: zmd->devname, q: devname);
2876	zmd->dev = dev;
2877	zmd->nr_devs = num_dev;
2878	zmd->mblk_rbtree = RB_ROOT;
2879	init_rwsem(&zmd->mblk_sem);
2880	mutex_init(&zmd->mblk_flush_lock);
2881	spin_lock_init(&zmd->mblk_lock);
2882	INIT_LIST_HEAD(list: &zmd->mblk_lru_list);
2883	INIT_LIST_HEAD(list: &zmd->mblk_dirty_list);
2884
2885	mutex_init(&zmd->map_lock);
2886
2887	atomic_set(v: &zmd->unmap_nr_cache, i: 0);
2888	INIT_LIST_HEAD(list: &zmd->unmap_cache_list);
2889	INIT_LIST_HEAD(list: &zmd->map_cache_list);
2890
2891	atomic_set(v: &zmd->nr_reserved_seq_zones, i: 0);
2892	INIT_LIST_HEAD(list: &zmd->reserved_seq_zones_list);
2893
2894	init_waitqueue_head(&zmd->free_wq);
2895
2896	/* Initialize zone descriptors */
2897	ret = dmz_init_zones(zmd);
2898	if (ret)
2899	goto err;
2900
2901	/* Get super block */
2902	ret = dmz_load_sb(zmd);
2903	if (ret)
2904	goto err;
2905
2906	/* Set metadata zones starting from sb_zone */
2907	for (i = 0; i < zmd->nr_meta_zones << 1; i++) {
2908	zone = dmz_get(zmd, zone_id: zmd->sb[0].zone->id + i);
2909	if (!zone) {
2910	dmz_zmd_err(zmd,
2911	"metadata zone %u not present", i);
2912	ret = -ENXIO;
2913	goto err;
2914	}
2915	if (!dmz_is_rnd(zone) && !dmz_is_cache(zone)) {
2916	dmz_zmd_err(zmd,
2917	"metadata zone %d is not random", i);
2918	ret = -ENXIO;
2919	goto err;
2920	}
2921	set_bit(nr: DMZ_META, addr: &zone->flags);
2922	}
2923	/* Load mapping table */
2924	ret = dmz_load_mapping(zmd);
2925	if (ret)
2926	goto err;
2927
2928	/*
2929	* Cache size boundaries: allow at least 2 super blocks, the chunk map
2930	* blocks and enough blocks to be able to cache the bitmap blocks of
2931	* up to 16 zones when idle (min_nr_mblks). Otherwise, if busy, allow
2932	* the cache to add 512 more metadata blocks.
2933	*/
2934	zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;
2935	zmd->max_nr_mblks = zmd->min_nr_mblks + 512;
2936
2937	/* Metadata cache shrinker */
2938	zmd->mblk_shrinker = shrinker_alloc(flags: 0, fmt: "dm-zoned-meta:(%u:%u)",
2939	MAJOR(dev->bdev->bd_dev),
2940	MINOR(dev->bdev->bd_dev));
2941	if (!zmd->mblk_shrinker) {
2942	ret = -ENOMEM;
2943	dmz_zmd_err(zmd, "Allocate metadata cache shrinker failed");
2944	goto err;
2945	}
2946
2947	zmd->mblk_shrinker->count_objects = dmz_mblock_shrinker_count;
2948	zmd->mblk_shrinker->scan_objects = dmz_mblock_shrinker_scan;
2949	zmd->mblk_shrinker->private_data = zmd;
2950
2951	shrinker_register(shrinker: zmd->mblk_shrinker);
2952
2953	dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);
2954	for (i = 0; i < zmd->nr_devs; i++)
2955	dmz_print_dev(zmd, num: i);
2956
2957	dmz_zmd_info(zmd, " %u zones of %llu 512-byte logical sectors",
2958	zmd->nr_zones, (u64)zmd->zone_nr_sectors);
2959	dmz_zmd_debug(zmd, " %u metadata zones",
2960	zmd->nr_meta_zones * 2);
2961	dmz_zmd_debug(zmd, " %u data zones for %u chunks",
2962	zmd->nr_data_zones, zmd->nr_chunks);
2963	dmz_zmd_debug(zmd, " %u cache zones (%u unmapped)",
2964	zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));
2965	for (i = 0; i < zmd->nr_devs; i++) {
2966	dmz_zmd_debug(zmd, " %u random zones (%u unmapped)",
2967	dmz_nr_rnd_zones(zmd, i),
2968	dmz_nr_unmap_rnd_zones(zmd, i));
2969	dmz_zmd_debug(zmd, " %u sequential zones (%u unmapped)",
2970	dmz_nr_seq_zones(zmd, i),
2971	dmz_nr_unmap_seq_zones(zmd, i));
2972	}
2973	dmz_zmd_debug(zmd, " %u reserved sequential data zones",
2974	zmd->nr_reserved_seq);
2975	dmz_zmd_debug(zmd, "Format:");
2976	dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",
2977	zmd->nr_meta_blocks, zmd->max_nr_mblks);
2978	dmz_zmd_debug(zmd, " %u data zone mapping blocks",
2979	zmd->nr_map_blocks);
2980	dmz_zmd_debug(zmd, " %u bitmap blocks",
2981	zmd->nr_bitmap_blocks);
2982
2983	*metadata = zmd;
2984
2985	return 0;
2986	err:
2987	dmz_cleanup_metadata(zmd);
2988	kfree(objp: zmd);
2989	*metadata = NULL;
2990
2991	return ret;
2992	}
2993
2994	/*
2995	* Cleanup the zoned metadata resources.
2996	*/
2997	void dmz_dtr_metadata(struct dmz_metadata *zmd)
2998	{
2999	shrinker_free(shrinker: zmd->mblk_shrinker);
3000	dmz_cleanup_metadata(zmd);
3001	kfree(objp: zmd);
3002	}
3003