1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2011-2012 Red Hat UK.
4	*
5	* This file is released under the GPL.
6	*/
7
8	#include "dm-thin-metadata.h"
9	#include "dm-bio-prison-v1.h"
10	#include "dm.h"
11
12	#include <linux/device-mapper.h>
13	#include <linux/dm-io.h>
14	#include <linux/dm-kcopyd.h>
15	#include <linux/jiffies.h>
16	#include <linux/log2.h>
17	#include <linux/list.h>
18	#include <linux/rculist.h>
19	#include <linux/init.h>
20	#include <linux/module.h>
21	#include <linux/slab.h>
22	#include <linux/vmalloc.h>
23	#include <linux/sort.h>
24	#include <linux/rbtree.h>
25
26	#define DM_MSG_PREFIX "thin"
27
28	/*
29	* Tunable constants
30	*/
31	#define ENDIO_HOOK_POOL_SIZE 1024
32	#define MAPPING_POOL_SIZE 1024
33	#define COMMIT_PERIOD HZ
34	#define NO_SPACE_TIMEOUT_SECS 60
35
36	static unsigned int no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
37
38	DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
39	"A percentage of time allocated for copy on write");
40
41	/*
42	* The block size of the device holding pool data must be
43	* between 64KB and 1GB.
44	*/
45	#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
46	#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
47
48	/*
49	* Device id is restricted to 24 bits.
50	*/
51	#define MAX_DEV_ID ((1 << 24) - 1)
52
53	/*
54	* How do we handle breaking sharing of data blocks?
55	* =================================================
56	*
57	* We use a standard copy-on-write btree to store the mappings for the
58	* devices (note I'm talking about copy-on-write of the metadata here, not
59	* the data). When you take an internal snapshot you clone the root node
60	* of the origin btree. After this there is no concept of an origin or a
61	* snapshot. They are just two device trees that happen to point to the
62	* same data blocks.
63	*
64	* When we get a write in we decide if it's to a shared data block using
65	* some timestamp magic. If it is, we have to break sharing.
66	*
67	* Let's say we write to a shared block in what was the origin. The
68	* steps are:
69	*
70	* i) plug io further to this physical block. (see bio_prison code).
71	*
72	* ii) quiesce any read io to that shared data block. Obviously
73	* including all devices that share this block. (see dm_deferred_set code)
74	*
75	* iii) copy the data block to a newly allocate block. This step can be
76	* missed out if the io covers the block. (schedule_copy).
77	*
78	* iv) insert the new mapping into the origin's btree
79	* (process_prepared_mapping). This act of inserting breaks some
80	* sharing of btree nodes between the two devices. Breaking sharing only
81	* effects the btree of that specific device. Btrees for the other
82	* devices that share the block never change. The btree for the origin
83	* device as it was after the last commit is untouched, ie. we're using
84	* persistent data structures in the functional programming sense.
85	*
86	* v) unplug io to this physical block, including the io that triggered
87	* the breaking of sharing.
88	*
89	* Steps (ii) and (iii) occur in parallel.
90	*
91	* The metadata _doesn't_ need to be committed before the io continues. We
92	* get away with this because the io is always written to a _new_ block.
93	* If there's a crash, then:
94	*
95	* - The origin mapping will point to the old origin block (the shared
96	* one). This will contain the data as it was before the io that triggered
97	* the breaking of sharing came in.
98	*
99	* - The snap mapping still points to the old block. As it would after
100	* the commit.
101	*
102	* The downside of this scheme is the timestamp magic isn't perfect, and
103	* will continue to think that data block in the snapshot device is shared
104	* even after the write to the origin has broken sharing. I suspect data
105	* blocks will typically be shared by many different devices, so we're
106	* breaking sharing n + 1 times, rather than n, where n is the number of
107	* devices that reference this data block. At the moment I think the
108	* benefits far, far outweigh the disadvantages.
109	*/
110
111	/*----------------------------------------------------------------*/
112
113	/*
114	* Key building.
115	*/
116	enum lock_space {
117	VIRTUAL,
118	PHYSICAL
119	};
120
121	static bool build_key(struct dm_thin_device *td, enum lock_space ls,
122	dm_block_t b, dm_block_t e, struct dm_cell_key *key)
123	{
124	key->virtual = (ls == VIRTUAL);
125	key->dev = dm_thin_dev_id(td);
126	key->block_begin = b;
127	key->block_end = e;
128
129	return dm_cell_key_has_valid_range(key);
130	}
131
132	static void build_data_key(struct dm_thin_device *td, dm_block_t b,
133	struct dm_cell_key *key)
134	{
135	(void) build_key(td, ls: PHYSICAL, b, e: b + 1llu, key);
136	}
137
138	static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
139	struct dm_cell_key *key)
140	{
141	(void) build_key(td, ls: VIRTUAL, b, e: b + 1llu, key);
142	}
143
144	/*----------------------------------------------------------------*/
145
146	#define THROTTLE_THRESHOLD (1 * HZ)
147
148	struct throttle {
149	struct rw_semaphore lock;
150	unsigned long threshold;
151	bool throttle_applied;
152	};
153
154	static void throttle_init(struct throttle *t)
155	{
156	init_rwsem(&t->lock);
157	t->throttle_applied = false;
158	}
159
160	static void throttle_work_start(struct throttle *t)
161	{
162	t->threshold = jiffies + THROTTLE_THRESHOLD;
163	}
164
165	static void throttle_work_update(struct throttle *t)
166	{
167	if (!t->throttle_applied && time_is_before_jiffies(t->threshold)) {
168	down_write(sem: &t->lock);
169	t->throttle_applied = true;
170	}
171	}
172
173	static void throttle_work_complete(struct throttle *t)
174	{
175	if (t->throttle_applied) {
176	t->throttle_applied = false;
177	up_write(sem: &t->lock);
178	}
179	}
180
181	static void throttle_lock(struct throttle *t)
182	{
183	down_read(sem: &t->lock);
184	}
185
186	static void throttle_unlock(struct throttle *t)
187	{
188	up_read(sem: &t->lock);
189	}
190
191	/*----------------------------------------------------------------*/
192
193	/*
194	* A pool device ties together a metadata device and a data device. It
195	* also provides the interface for creating and destroying internal
196	* devices.
197	*/
198	struct dm_thin_new_mapping;
199
200	/*
201	* The pool runs in various modes. Ordered in degraded order for comparisons.
202	*/
203	enum pool_mode {
204	PM_WRITE, /* metadata may be changed */
205	PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */
206
207	/*
208	* Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY.
209	*/
210	PM_OUT_OF_METADATA_SPACE,
211	PM_READ_ONLY, /* metadata may not be changed */
212
213	PM_FAIL, /* all I/O fails */
214	};
215
216	struct pool_features {
217	enum pool_mode mode;
218
219	bool zero_new_blocks:1;
220	bool discard_enabled:1;
221	bool discard_passdown:1;
222	bool error_if_no_space:1;
223	};
224
225	struct thin_c;
226	typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
227	typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
228	typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
229
230	#define CELL_SORT_ARRAY_SIZE 8192
231
232	struct pool {
233	struct list_head list;
234	struct dm_target *ti; /* Only set if a pool target is bound */
235
236	struct mapped_device *pool_md;
237	struct block_device *data_dev;
238	struct block_device *md_dev;
239	struct dm_pool_metadata *pmd;
240
241	dm_block_t low_water_blocks;
242	uint32_t sectors_per_block;
243	int sectors_per_block_shift;
244
245	struct pool_features pf;
246	bool low_water_triggered:1; /* A dm event has been sent */
247	bool suspended:1;
248	bool out_of_data_space:1;
249
250	struct dm_bio_prison *prison;
251	struct dm_kcopyd_client *copier;
252
253	struct work_struct worker;
254	struct workqueue_struct *wq;
255	struct throttle throttle;
256	struct delayed_work waker;
257	struct delayed_work no_space_timeout;
258
259	unsigned long last_commit_jiffies;
260	unsigned int ref_count;
261
262	spinlock_t lock;
263	struct bio_list deferred_flush_bios;
264	struct bio_list deferred_flush_completions;
265	struct list_head prepared_mappings;
266	struct list_head prepared_discards;
267	struct list_head prepared_discards_pt2;
268	struct list_head active_thins;
269
270	struct dm_deferred_set *shared_read_ds;
271	struct dm_deferred_set *all_io_ds;
272
273	struct dm_thin_new_mapping *next_mapping;
274
275	process_bio_fn process_bio;
276	process_bio_fn process_discard;
277
278	process_cell_fn process_cell;
279	process_cell_fn process_discard_cell;
280
281	process_mapping_fn process_prepared_mapping;
282	process_mapping_fn process_prepared_discard;
283	process_mapping_fn process_prepared_discard_pt2;
284
285	struct dm_bio_prison_cell **cell_sort_array;
286
287	mempool_t mapping_pool;
288	};
289
290	static void metadata_operation_failed(struct pool *pool, const char *op, int r);
291
292	static enum pool_mode get_pool_mode(struct pool *pool)
293	{
294	return pool->pf.mode;
295	}
296
297	static void notify_of_pool_mode_change(struct pool *pool)
298	{
299	static const char *descs[] = {
300	"write",
301	"out-of-data-space",
302	"read-only",
303	"read-only",
304	"fail"
305	};
306	const char *extra_desc = NULL;
307	enum pool_mode mode = get_pool_mode(pool);
308
309	if (mode == PM_OUT_OF_DATA_SPACE) {
310	if (!pool->pf.error_if_no_space)
311	extra_desc = " (queue IO)";
312	else
313	extra_desc = " (error IO)";
314	}
315
316	dm_table_event(t: pool->ti->table);
317	DMINFO("%s: switching pool to %s%s mode",
318	dm_device_name(pool->pool_md),
319	descs[(int)mode], extra_desc ? : "");
320	}
321
322	/*
323	* Target context for a pool.
324	*/
325	struct pool_c {
326	struct dm_target *ti;
327	struct pool *pool;
328	struct dm_dev *data_dev;
329	struct dm_dev *metadata_dev;
330
331	dm_block_t low_water_blocks;
332	struct pool_features requested_pf; /* Features requested during table load */
333	struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */
334	};
335
336	/*
337	* Target context for a thin.
338	*/
339	struct thin_c {
340	struct list_head list;
341	struct dm_dev *pool_dev;
342	struct dm_dev *origin_dev;
343	sector_t origin_size;
344	dm_thin_id dev_id;
345
346	struct pool *pool;
347	struct dm_thin_device *td;
348	struct mapped_device *thin_md;
349
350	bool requeue_mode:1;
351	spinlock_t lock;
352	struct list_head deferred_cells;
353	struct bio_list deferred_bio_list;
354	struct bio_list retry_on_resume_list;
355	struct rb_root sort_bio_list; /* sorted list of deferred bios */
356
357	/*
358	* Ensures the thin is not destroyed until the worker has finished
359	* iterating the active_thins list.
360	*/
361	refcount_t refcount;
362	struct completion can_destroy;
363	};
364
365	/*----------------------------------------------------------------*/
366
367	static bool block_size_is_power_of_two(struct pool *pool)
368	{
369	return pool->sectors_per_block_shift >= 0;
370	}
371
372	static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
373	{
374	return block_size_is_power_of_two(pool) ?
375	(b << pool->sectors_per_block_shift) :
376	(b * pool->sectors_per_block);
377	}
378
379	/*----------------------------------------------------------------*/
380
381	struct discard_op {
382	struct thin_c *tc;
383	struct blk_plug plug;
384	struct bio *parent_bio;
385	struct bio *bio;
386	};
387
388	static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent)
389	{
390	BUG_ON(!parent);
391
392	op->tc = tc;
393	blk_start_plug(&op->plug);
394	op->parent_bio = parent;
395	op->bio = NULL;
396	}
397
398	static void issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e)
399	{
400	struct thin_c *tc = op->tc;
401	sector_t s = block_to_sectors(pool: tc->pool, b: data_b);
402	sector_t len = block_to_sectors(pool: tc->pool, b: data_e - data_b);
403
404	__blkdev_issue_discard(bdev: tc->pool_dev->bdev, sector: s, nr_sects: len, GFP_NOIO, biop: &op->bio);
405	}
406
407	static void end_discard(struct discard_op *op, int r)
408	{
409	if (op->bio) {
410	/*
411	* Even if one of the calls to issue_discard failed, we
412	* need to wait for the chain to complete.
413	*/
414	bio_chain(op->bio, op->parent_bio);
415	op->bio->bi_opf = REQ_OP_DISCARD;
416	submit_bio(bio: op->bio);
417	}
418
419	blk_finish_plug(&op->plug);
420
421	/*
422	* Even if r is set, there could be sub discards in flight that we
423	* need to wait for.
424	*/
425	if (r && !op->parent_bio->bi_status)
426	op->parent_bio->bi_status = errno_to_blk_status(errno: r);
427	bio_endio(op->parent_bio);
428	}
429
430	/*----------------------------------------------------------------*/
431
432	/*
433	* wake_worker() is used when new work is queued and when pool_resume is
434	* ready to continue deferred IO processing.
435	*/
436	static void wake_worker(struct pool *pool)
437	{
438	queue_work(wq: pool->wq, work: &pool->worker);
439	}
440
441	/*----------------------------------------------------------------*/
442
443	static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
444	struct dm_bio_prison_cell **cell_result)
445	{
446	int r;
447	struct dm_bio_prison_cell *cell_prealloc;
448
449	/*
450	* Allocate a cell from the prison's mempool.
451	* This might block but it can't fail.
452	*/
453	cell_prealloc = dm_bio_prison_alloc_cell(prison: pool->prison, GFP_NOIO);
454
455	r = dm_bio_detain(prison: pool->prison, key, inmate: bio, cell_prealloc, cell_result);
456	if (r) {
457	/*
458	* We reused an old cell; we can get rid of
459	* the new one.
460	*/
461	dm_bio_prison_free_cell(prison: pool->prison, cell: cell_prealloc);
462	}
463
464	return r;
465	}
466
467	static void cell_release(struct pool *pool,
468	struct dm_bio_prison_cell *cell,
469	struct bio_list *bios)
470	{
471	dm_cell_release(prison: pool->prison, cell, bios);
472	dm_bio_prison_free_cell(prison: pool->prison, cell);
473	}
474
475	static void cell_visit_release(struct pool *pool,
476	void (*fn)(void *, struct dm_bio_prison_cell *),
477	void *context,
478	struct dm_bio_prison_cell *cell)
479	{
480	dm_cell_visit_release(prison: pool->prison, visit_fn: fn, context, cell);
481	dm_bio_prison_free_cell(prison: pool->prison, cell);
482	}
483
484	static void cell_release_no_holder(struct pool *pool,
485	struct dm_bio_prison_cell *cell,
486	struct bio_list *bios)
487	{
488	dm_cell_release_no_holder(prison: pool->prison, cell, inmates: bios);
489	dm_bio_prison_free_cell(prison: pool->prison, cell);
490	}
491
492	static void cell_error_with_code(struct pool *pool,
493	struct dm_bio_prison_cell *cell, blk_status_t error_code)
494	{
495	dm_cell_error(prison: pool->prison, cell, error: error_code);
496	dm_bio_prison_free_cell(prison: pool->prison, cell);
497	}
498
499	static blk_status_t get_pool_io_error_code(struct pool *pool)
500	{
501	return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR;
502	}
503
504	static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
505	{
506	cell_error_with_code(pool, cell, error_code: get_pool_io_error_code(pool));
507	}
508
509	static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
510	{
511	cell_error_with_code(pool, cell, error_code: 0);
512	}
513
514	static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
515	{
516	cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE);
517	}
518
519	/*----------------------------------------------------------------*/
520
521	/*
522	* A global list of pools that uses a struct mapped_device as a key.
523	*/
524	static struct dm_thin_pool_table {
525	struct mutex mutex;
526	struct list_head pools;
527	} dm_thin_pool_table;
528
529	static void pool_table_init(void)
530	{
531	mutex_init(&dm_thin_pool_table.mutex);
532	INIT_LIST_HEAD(list: &dm_thin_pool_table.pools);
533	}
534
535	static void pool_table_exit(void)
536	{
537	mutex_destroy(lock: &dm_thin_pool_table.mutex);
538	}
539
540	static void __pool_table_insert(struct pool *pool)
541	{
542	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
543	list_add(new: &pool->list, head: &dm_thin_pool_table.pools);
544	}
545
546	static void __pool_table_remove(struct pool *pool)
547	{
548	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
549	list_del(entry: &pool->list);
550	}
551
552	static struct pool *__pool_table_lookup(struct mapped_device *md)
553	{
554	struct pool *pool = NULL, *tmp;
555
556	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
557
558	list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
559	if (tmp->pool_md == md) {
560	pool = tmp;
561	break;
562	}
563	}
564
565	return pool;
566	}
567
568	static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
569	{
570	struct pool *pool = NULL, *tmp;
571
572	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
573
574	list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
575	if (tmp->md_dev == md_dev) {
576	pool = tmp;
577	break;
578	}
579	}
580
581	return pool;
582	}
583
584	/*----------------------------------------------------------------*/
585
586	struct dm_thin_endio_hook {
587	struct thin_c *tc;
588	struct dm_deferred_entry *shared_read_entry;
589	struct dm_deferred_entry *all_io_entry;
590	struct dm_thin_new_mapping *overwrite_mapping;
591	struct rb_node rb_node;
592	struct dm_bio_prison_cell *cell;
593	};
594
595	static void error_bio_list(struct bio_list *bios, blk_status_t error)
596	{
597	struct bio *bio;
598
599	while ((bio = bio_list_pop(bl: bios))) {
600	bio->bi_status = error;
601	bio_endio(bio);
602	}
603	}
604
605	static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master,
606	blk_status_t error)
607	{
608	struct bio_list bios;
609
610	bio_list_init(bl: &bios);
611
612	spin_lock_irq(lock: &tc->lock);
613	bio_list_merge_init(bl: &bios, bl2: master);
614	spin_unlock_irq(lock: &tc->lock);
615
616	error_bio_list(bios: &bios, error);
617	}
618
619	static void requeue_deferred_cells(struct thin_c *tc)
620	{
621	struct pool *pool = tc->pool;
622	struct list_head cells;
623	struct dm_bio_prison_cell *cell, *tmp;
624
625	INIT_LIST_HEAD(list: &cells);
626
627	spin_lock_irq(lock: &tc->lock);
628	list_splice_init(list: &tc->deferred_cells, head: &cells);
629	spin_unlock_irq(lock: &tc->lock);
630
631	list_for_each_entry_safe(cell, tmp, &cells, user_list)
632	cell_requeue(pool, cell);
633	}
634
635	static void requeue_io(struct thin_c *tc)
636	{
637	struct bio_list bios;
638
639	bio_list_init(bl: &bios);
640
641	spin_lock_irq(lock: &tc->lock);
642	bio_list_merge_init(bl: &bios, bl2: &tc->deferred_bio_list);
643	bio_list_merge_init(bl: &bios, bl2: &tc->retry_on_resume_list);
644	spin_unlock_irq(lock: &tc->lock);
645
646	error_bio_list(bios: &bios, BLK_STS_DM_REQUEUE);
647	requeue_deferred_cells(tc);
648	}
649
650	static void error_retry_list_with_code(struct pool *pool, blk_status_t error)
651	{
652	struct thin_c *tc;
653
654	rcu_read_lock();
655	list_for_each_entry_rcu(tc, &pool->active_thins, list)
656	error_thin_bio_list(tc, master: &tc->retry_on_resume_list, error);
657	rcu_read_unlock();
658	}
659
660	static void error_retry_list(struct pool *pool)
661	{
662	error_retry_list_with_code(pool, error: get_pool_io_error_code(pool));
663	}
664
665	/*
666	* This section of code contains the logic for processing a thin device's IO.
667	* Much of the code depends on pool object resources (lists, workqueues, etc)
668	* but most is exclusively called from the thin target rather than the thin-pool
669	* target.
670	*/
671
672	static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
673	{
674	struct pool *pool = tc->pool;
675	sector_t block_nr = bio->bi_iter.bi_sector;
676
677	if (block_size_is_power_of_two(pool))
678	block_nr >>= pool->sectors_per_block_shift;
679	else
680	(void) sector_div(block_nr, pool->sectors_per_block);
681
682	return block_nr;
683	}
684
685	/*
686	* Returns the _complete_ blocks that this bio covers.
687	*/
688	static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
689	dm_block_t *begin, dm_block_t *end)
690	{
691	struct pool *pool = tc->pool;
692	sector_t b = bio->bi_iter.bi_sector;
693	sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
694
695	b += pool->sectors_per_block - 1ull; /* so we round up */
696
697	if (block_size_is_power_of_two(pool)) {
698	b >>= pool->sectors_per_block_shift;
699	e >>= pool->sectors_per_block_shift;
700	} else {
701	(void) sector_div(b, pool->sectors_per_block);
702	(void) sector_div(e, pool->sectors_per_block);
703	}
704
705	if (e < b) {
706	/* Can happen if the bio is within a single block. */
707	e = b;
708	}
709
710	*begin = b;
711	*end = e;
712	}
713
714	static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
715	{
716	struct pool *pool = tc->pool;
717	sector_t bi_sector = bio->bi_iter.bi_sector;
718
719	bio_set_dev(bio, bdev: tc->pool_dev->bdev);
720	if (block_size_is_power_of_two(pool)) {
721	bio->bi_iter.bi_sector =
722	(block << pool->sectors_per_block_shift) |
723	(bi_sector & (pool->sectors_per_block - 1));
724	} else {
725	bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
726	sector_div(bi_sector, pool->sectors_per_block);
727	}
728	}
729
730	static void remap_to_origin(struct thin_c *tc, struct bio *bio)
731	{
732	bio_set_dev(bio, bdev: tc->origin_dev->bdev);
733	}
734
735	static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
736	{
737	return op_is_flush(op: bio->bi_opf) &&
738	dm_thin_changed_this_transaction(td: tc->td);
739	}
740
741	static void inc_all_io_entry(struct pool *pool, struct bio *bio)
742	{
743	struct dm_thin_endio_hook *h;
744
745	if (bio_op(bio) == REQ_OP_DISCARD)
746	return;
747
748	h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
749	h->all_io_entry = dm_deferred_entry_inc(ds: pool->all_io_ds);
750	}
751
752	static void issue(struct thin_c *tc, struct bio *bio)
753	{
754	struct pool *pool = tc->pool;
755
756	if (!bio_triggers_commit(tc, bio)) {
757	dm_submit_bio_remap(clone: bio, NULL);
758	return;
759	}
760
761	/*
762	* Complete bio with an error if earlier I/O caused changes to
763	* the metadata that can't be committed e.g, due to I/O errors
764	* on the metadata device.
765	*/
766	if (dm_thin_aborted_changes(td: tc->td)) {
767	bio_io_error(bio);
768	return;
769	}
770
771	/*
772	* Batch together any bios that trigger commits and then issue a
773	* single commit for them in process_deferred_bios().
774	*/
775	spin_lock_irq(lock: &pool->lock);
776	bio_list_add(bl: &pool->deferred_flush_bios, bio);
777	spin_unlock_irq(lock: &pool->lock);
778	}
779
780	static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
781	{
782	remap_to_origin(tc, bio);
783	issue(tc, bio);
784	}
785
786	static void remap_and_issue(struct thin_c *tc, struct bio *bio,
787	dm_block_t block)
788	{
789	remap(tc, bio, block);
790	issue(tc, bio);
791	}
792
793	/*----------------------------------------------------------------*/
794
795	/*
796	* Bio endio functions.
797	*/
798	struct dm_thin_new_mapping {
799	struct list_head list;
800
801	bool pass_discard:1;
802	bool maybe_shared:1;
803
804	/*
805	* Track quiescing, copying and zeroing preparation actions. When this
806	* counter hits zero the block is prepared and can be inserted into the
807	* btree.
808	*/
809	atomic_t prepare_actions;
810
811	blk_status_t status;
812	struct thin_c *tc;
813	dm_block_t virt_begin, virt_end;
814	dm_block_t data_block;
815	struct dm_bio_prison_cell *cell;
816
817	/*
818	* If the bio covers the whole area of a block then we can avoid
819	* zeroing or copying. Instead this bio is hooked. The bio will
820	* still be in the cell, so care has to be taken to avoid issuing
821	* the bio twice.
822	*/
823	struct bio *bio;
824	bio_end_io_t *saved_bi_end_io;
825	};
826
827	static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
828	{
829	struct pool *pool = m->tc->pool;
830
831	if (atomic_dec_and_test(v: &m->prepare_actions)) {
832	list_add_tail(new: &m->list, head: &pool->prepared_mappings);
833	wake_worker(pool);
834	}
835	}
836
837	static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
838	{
839	unsigned long flags;
840	struct pool *pool = m->tc->pool;
841
842	spin_lock_irqsave(&pool->lock, flags);
843	__complete_mapping_preparation(m);
844	spin_unlock_irqrestore(lock: &pool->lock, flags);
845	}
846
847	static void copy_complete(int read_err, unsigned long write_err, void *context)
848	{
849	struct dm_thin_new_mapping *m = context;
850
851	m->status = read_err || write_err ? BLK_STS_IOERR : 0;
852	complete_mapping_preparation(m);
853	}
854
855	static void overwrite_endio(struct bio *bio)
856	{
857	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
858	struct dm_thin_new_mapping *m = h->overwrite_mapping;
859
860	bio->bi_end_io = m->saved_bi_end_io;
861
862	m->status = bio->bi_status;
863	complete_mapping_preparation(m);
864	}
865
866	/*----------------------------------------------------------------*/
867
868	/*
869	* Workqueue.
870	*/
871
872	/*
873	* Prepared mapping jobs.
874	*/
875
876	/*
877	* This sends the bios in the cell, except the original holder, back
878	* to the deferred_bios list.
879	*/
880	static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
881	{
882	struct pool *pool = tc->pool;
883	unsigned long flags;
884	struct bio_list bios;
885
886	bio_list_init(bl: &bios);
887	cell_release_no_holder(pool, cell, bios: &bios);
888
889	if (!bio_list_empty(bl: &bios)) {
890	spin_lock_irqsave(&tc->lock, flags);
891	bio_list_merge(bl: &tc->deferred_bio_list, bl2: &bios);
892	spin_unlock_irqrestore(lock: &tc->lock, flags);
893	wake_worker(pool);
894	}
895	}
896
897	static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
898
899	struct remap_info {
900	struct thin_c *tc;
901	struct bio_list defer_bios;
902	struct bio_list issue_bios;
903	};
904
905	static void __inc_remap_and_issue_cell(void *context,
906	struct dm_bio_prison_cell *cell)
907	{
908	struct remap_info *info = context;
909	struct bio *bio;
910
911	while ((bio = bio_list_pop(bl: &cell->bios))) {
912	if (op_is_flush(op: bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD)
913	bio_list_add(bl: &info->defer_bios, bio);
914	else {
915	inc_all_io_entry(pool: info->tc->pool, bio);
916
917	/*
918	* We can't issue the bios with the bio prison lock
919	* held, so we add them to a list to issue on
920	* return from this function.
921	*/
922	bio_list_add(bl: &info->issue_bios, bio);
923	}
924	}
925	}
926
927	static void inc_remap_and_issue_cell(struct thin_c *tc,
928	struct dm_bio_prison_cell *cell,
929	dm_block_t block)
930	{
931	struct bio *bio;
932	struct remap_info info;
933
934	info.tc = tc;
935	bio_list_init(bl: &info.defer_bios);
936	bio_list_init(bl: &info.issue_bios);
937
938	/*
939	* We have to be careful to inc any bios we're about to issue
940	* before the cell is released, and avoid a race with new bios
941	* being added to the cell.
942	*/
943	cell_visit_release(pool: tc->pool, fn: __inc_remap_and_issue_cell,
944	context: &info, cell);
945
946	while ((bio = bio_list_pop(bl: &info.defer_bios)))
947	thin_defer_bio(tc, bio);
948
949	while ((bio = bio_list_pop(bl: &info.issue_bios)))
950	remap_and_issue(tc: info.tc, bio, block);
951	}
952
953	static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
954	{
955	cell_error(pool: m->tc->pool, cell: m->cell);
956	list_del(entry: &m->list);
957	mempool_free(element: m, pool: &m->tc->pool->mapping_pool);
958	}
959
960	static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio)
961	{
962	struct pool *pool = tc->pool;
963
964	/*
965	* If the bio has the REQ_FUA flag set we must commit the metadata
966	* before signaling its completion.
967	*/
968	if (!bio_triggers_commit(tc, bio)) {
969	bio_endio(bio);
970	return;
971	}
972
973	/*
974	* Complete bio with an error if earlier I/O caused changes to the
975	* metadata that can't be committed, e.g, due to I/O errors on the
976	* metadata device.
977	*/
978	if (dm_thin_aborted_changes(td: tc->td)) {
979	bio_io_error(bio);
980	return;
981	}
982
983	/*
984	* Batch together any bios that trigger commits and then issue a
985	* single commit for them in process_deferred_bios().
986	*/
987	spin_lock_irq(lock: &pool->lock);
988	bio_list_add(bl: &pool->deferred_flush_completions, bio);
989	spin_unlock_irq(lock: &pool->lock);
990	}
991
992	static void process_prepared_mapping(struct dm_thin_new_mapping *m)
993	{
994	struct thin_c *tc = m->tc;
995	struct pool *pool = tc->pool;
996	struct bio *bio = m->bio;
997	int r;
998
999	if (m->status) {
1000	cell_error(pool, cell: m->cell);
1001	goto out;
1002	}
1003
1004	/*
1005	* Commit the prepared block into the mapping btree.
1006	* Any I/O for this block arriving after this point will get
1007	* remapped to it directly.
1008	*/
1009	r = dm_thin_insert_block(td: tc->td, block: m->virt_begin, data_block: m->data_block);
1010	if (r) {
1011	metadata_operation_failed(pool, op: "dm_thin_insert_block", r);
1012	cell_error(pool, cell: m->cell);
1013	goto out;
1014	}
1015
1016	/*
1017	* Release any bios held while the block was being provisioned.
1018	* If we are processing a write bio that completely covers the block,
1019	* we already processed it so can ignore it now when processing
1020	* the bios in the cell.
1021	*/
1022	if (bio) {
1023	inc_remap_and_issue_cell(tc, cell: m->cell, block: m->data_block);
1024	complete_overwrite_bio(tc, bio);
1025	} else {
1026	inc_all_io_entry(pool: tc->pool, bio: m->cell->holder);
1027	remap_and_issue(tc, bio: m->cell->holder, block: m->data_block);
1028	inc_remap_and_issue_cell(tc, cell: m->cell, block: m->data_block);
1029	}
1030
1031	out:
1032	list_del(entry: &m->list);
1033	mempool_free(element: m, pool: &pool->mapping_pool);
1034	}
1035
1036	/*----------------------------------------------------------------*/
1037
1038	static void free_discard_mapping(struct dm_thin_new_mapping *m)
1039	{
1040	struct thin_c *tc = m->tc;
1041
1042	if (m->cell)
1043	cell_defer_no_holder(tc, cell: m->cell);
1044	mempool_free(element: m, pool: &tc->pool->mapping_pool);
1045	}
1046
1047	static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
1048	{
1049	bio_io_error(bio: m->bio);
1050	free_discard_mapping(m);
1051	}
1052
1053	static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
1054	{
1055	bio_endio(m->bio);
1056	free_discard_mapping(m);
1057	}
1058
1059	static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
1060	{
1061	int r;
1062	struct thin_c *tc = m->tc;
1063
1064	r = dm_thin_remove_range(td: tc->td, begin: m->cell->key.block_begin, end: m->cell->key.block_end);
1065	if (r) {
1066	metadata_operation_failed(pool: tc->pool, op: "dm_thin_remove_range", r);
1067	bio_io_error(bio: m->bio);
1068	} else
1069	bio_endio(m->bio);
1070
1071	cell_defer_no_holder(tc, cell: m->cell);
1072	mempool_free(element: m, pool: &tc->pool->mapping_pool);
1073	}
1074
1075	/*----------------------------------------------------------------*/
1076
1077	static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m,
1078	struct bio *discard_parent)
1079	{
1080	/*
1081	* We've already unmapped this range of blocks, but before we
1082	* passdown we have to check that these blocks are now unused.
1083	*/
1084	int r = 0;
1085	bool shared = true;
1086	struct thin_c *tc = m->tc;
1087	struct pool *pool = tc->pool;
1088	dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
1089	struct discard_op op;
1090
1091	begin_discard(op: &op, tc, parent: discard_parent);
1092	while (b != end) {
1093	/* find start of unmapped run */
1094	for (; b < end; b++) {
1095	r = dm_pool_block_is_shared(pmd: pool->pmd, b, result: &shared);
1096	if (r)
1097	goto out;
1098
1099	if (!shared)
1100	break;
1101	}
1102
1103	if (b == end)
1104	break;
1105
1106	/* find end of run */
1107	for (e = b + 1; e != end; e++) {
1108	r = dm_pool_block_is_shared(pmd: pool->pmd, b: e, result: &shared);
1109	if (r)
1110	goto out;
1111
1112	if (shared)
1113	break;
1114	}
1115
1116	issue_discard(op: &op, data_b: b, data_e: e);
1117
1118	b = e;
1119	}
1120	out:
1121	end_discard(op: &op, r);
1122	}
1123
1124	static void queue_passdown_pt2(struct dm_thin_new_mapping *m)
1125	{
1126	unsigned long flags;
1127	struct pool *pool = m->tc->pool;
1128
1129	spin_lock_irqsave(&pool->lock, flags);
1130	list_add_tail(new: &m->list, head: &pool->prepared_discards_pt2);
1131	spin_unlock_irqrestore(lock: &pool->lock, flags);
1132	wake_worker(pool);
1133	}
1134
1135	static void passdown_endio(struct bio *bio)
1136	{
1137	/*
1138	* It doesn't matter if the passdown discard failed, we still want
1139	* to unmap (we ignore err).
1140	*/
1141	queue_passdown_pt2(m: bio->bi_private);
1142	bio_put(bio);
1143	}
1144
1145	static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
1146	{
1147	int r;
1148	struct thin_c *tc = m->tc;
1149	struct pool *pool = tc->pool;
1150	struct bio *discard_parent;
1151	dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin);
1152
1153	/*
1154	* Only this thread allocates blocks, so we can be sure that the
1155	* newly unmapped blocks will not be allocated before the end of
1156	* the function.
1157	*/
1158	r = dm_thin_remove_range(td: tc->td, begin: m->virt_begin, end: m->virt_end);
1159	if (r) {
1160	metadata_operation_failed(pool, op: "dm_thin_remove_range", r);
1161	bio_io_error(bio: m->bio);
1162	cell_defer_no_holder(tc, cell: m->cell);
1163	mempool_free(element: m, pool: &pool->mapping_pool);
1164	return;
1165	}
1166
1167	/*
1168	* Increment the unmapped blocks. This prevents a race between the
1169	* passdown io and reallocation of freed blocks.
1170	*/
1171	r = dm_pool_inc_data_range(pmd: pool->pmd, b: m->data_block, e: data_end);
1172	if (r) {
1173	metadata_operation_failed(pool, op: "dm_pool_inc_data_range", r);
1174	bio_io_error(bio: m->bio);
1175	cell_defer_no_holder(tc, cell: m->cell);
1176	mempool_free(element: m, pool: &pool->mapping_pool);
1177	return;
1178	}
1179
1180	discard_parent = bio_alloc(NULL, nr_vecs: 1, opf: 0, GFP_NOIO);
1181	discard_parent->bi_end_io = passdown_endio;
1182	discard_parent->bi_private = m;
1183	if (m->maybe_shared)
1184	passdown_double_checking_shared_status(m, discard_parent);
1185	else {
1186	struct discard_op op;
1187
1188	begin_discard(op: &op, tc, parent: discard_parent);
1189	issue_discard(op: &op, data_b: m->data_block, data_e: data_end);
1190	end_discard(op: &op, r: 0);
1191	}
1192	}
1193
1194	static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)
1195	{
1196	int r;
1197	struct thin_c *tc = m->tc;
1198	struct pool *pool = tc->pool;
1199
1200	/*
1201	* The passdown has completed, so now we can decrement all those
1202	* unmapped blocks.
1203	*/
1204	r = dm_pool_dec_data_range(pmd: pool->pmd, b: m->data_block,
1205	e: m->data_block + (m->virt_end - m->virt_begin));
1206	if (r) {
1207	metadata_operation_failed(pool, op: "dm_pool_dec_data_range", r);
1208	bio_io_error(bio: m->bio);
1209	} else
1210	bio_endio(m->bio);
1211
1212	cell_defer_no_holder(tc, cell: m->cell);
1213	mempool_free(element: m, pool: &pool->mapping_pool);
1214	}
1215
1216	static void process_prepared(struct pool *pool, struct list_head *head,
1217	process_mapping_fn *fn)
1218	{
1219	struct list_head maps;
1220	struct dm_thin_new_mapping *m, *tmp;
1221
1222	INIT_LIST_HEAD(list: &maps);
1223	spin_lock_irq(lock: &pool->lock);
1224	list_splice_init(list: head, head: &maps);
1225	spin_unlock_irq(lock: &pool->lock);
1226
1227	list_for_each_entry_safe(m, tmp, &maps, list)
1228	(*fn)(m);
1229	}
1230
1231	/*
1232	* Deferred bio jobs.
1233	*/
1234	static int io_overlaps_block(struct pool *pool, struct bio *bio)
1235	{
1236	return bio->bi_iter.bi_size ==
1237	(pool->sectors_per_block << SECTOR_SHIFT);
1238	}
1239
1240	static int io_overwrites_block(struct pool *pool, struct bio *bio)
1241	{
1242	return (bio_data_dir(bio) == WRITE) &&
1243	io_overlaps_block(pool, bio);
1244	}
1245
1246	static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
1247	bio_end_io_t *fn)
1248	{
1249	*save = bio->bi_end_io;
1250	bio->bi_end_io = fn;
1251	}
1252
1253	static int ensure_next_mapping(struct pool *pool)
1254	{
1255	if (pool->next_mapping)
1256	return 0;
1257
1258	pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC);
1259
1260	return pool->next_mapping ? 0 : -ENOMEM;
1261	}
1262
1263	static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
1264	{
1265	struct dm_thin_new_mapping *m = pool->next_mapping;
1266
1267	BUG_ON(!pool->next_mapping);
1268
1269	memset(m, 0, sizeof(struct dm_thin_new_mapping));
1270	INIT_LIST_HEAD(list: &m->list);
1271	m->bio = NULL;
1272
1273	pool->next_mapping = NULL;
1274
1275	return m;
1276	}
1277
1278	static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
1279	sector_t begin, sector_t end)
1280	{
1281	struct dm_io_region to;
1282
1283	to.bdev = tc->pool_dev->bdev;
1284	to.sector = begin;
1285	to.count = end - begin;
1286
1287	dm_kcopyd_zero(kc: tc->pool->copier, num_dests: 1, dests: &to, flags: 0, fn: copy_complete, context: m);
1288	}
1289
1290	static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
1291	dm_block_t data_begin,
1292	struct dm_thin_new_mapping *m)
1293	{
1294	struct pool *pool = tc->pool;
1295	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
1296
1297	h->overwrite_mapping = m;
1298	m->bio = bio;
1299	save_and_set_endio(bio, save: &m->saved_bi_end_io, fn: overwrite_endio);
1300	inc_all_io_entry(pool, bio);
1301	remap_and_issue(tc, bio, block: data_begin);
1302	}
1303
1304	/*
1305	* A partial copy also needs to zero the uncopied region.
1306	*/
1307	static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
1308	struct dm_dev *origin, dm_block_t data_origin,
1309	dm_block_t data_dest,
1310	struct dm_bio_prison_cell *cell, struct bio *bio,
1311	sector_t len)
1312	{
1313	struct pool *pool = tc->pool;
1314	struct dm_thin_new_mapping *m = get_next_mapping(pool);
1315
1316	m->tc = tc;
1317	m->virt_begin = virt_block;
1318	m->virt_end = virt_block + 1u;
1319	m->data_block = data_dest;
1320	m->cell = cell;
1321
1322	/*
1323	* quiesce action + copy action + an extra reference held for the
1324	* duration of this function (we may need to inc later for a
1325	* partial zero).
1326	*/
1327	atomic_set(v: &m->prepare_actions, i: 3);
1328
1329	if (!dm_deferred_set_add_work(ds: pool->shared_read_ds, work: &m->list))
1330	complete_mapping_preparation(m); /* already quiesced */
1331
1332	/*
1333	* IO to pool_dev remaps to the pool target's data_dev.
1334	*
1335	* If the whole block of data is being overwritten, we can issue the
1336	* bio immediately. Otherwise we use kcopyd to clone the data first.
1337	*/
1338	if (io_overwrites_block(pool, bio))
1339	remap_and_issue_overwrite(tc, bio, data_begin: data_dest, m);
1340	else {
1341	struct dm_io_region from, to;
1342
1343	from.bdev = origin->bdev;
1344	from.sector = data_origin * pool->sectors_per_block;
1345	from.count = len;
1346
1347	to.bdev = tc->pool_dev->bdev;
1348	to.sector = data_dest * pool->sectors_per_block;
1349	to.count = len;
1350
1351	dm_kcopyd_copy(kc: pool->copier, from: &from, num_dests: 1, dests: &to,
1352	flags: 0, fn: copy_complete, context: m);
1353
1354	/*
1355	* Do we need to zero a tail region?
1356	*/
1357	if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
1358	atomic_inc(v: &m->prepare_actions);
1359	ll_zero(tc, m,
1360	begin: data_dest * pool->sectors_per_block + len,
1361	end: (data_dest + 1) * pool->sectors_per_block);
1362	}
1363	}
1364
1365	complete_mapping_preparation(m); /* drop our ref */
1366	}
1367
1368	static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
1369	dm_block_t data_origin, dm_block_t data_dest,
1370	struct dm_bio_prison_cell *cell, struct bio *bio)
1371	{
1372	schedule_copy(tc, virt_block, origin: tc->pool_dev,
1373	data_origin, data_dest, cell, bio,
1374	len: tc->pool->sectors_per_block);
1375	}
1376
1377	static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
1378	dm_block_t data_block, struct dm_bio_prison_cell *cell,
1379	struct bio *bio)
1380	{
1381	struct pool *pool = tc->pool;
1382	struct dm_thin_new_mapping *m = get_next_mapping(pool);
1383
1384	atomic_set(v: &m->prepare_actions, i: 1); /* no need to quiesce */
1385	m->tc = tc;
1386	m->virt_begin = virt_block;
1387	m->virt_end = virt_block + 1u;
1388	m->data_block = data_block;
1389	m->cell = cell;
1390
1391	/*
1392	* If the whole block of data is being overwritten or we are not
1393	* zeroing pre-existing data, we can issue the bio immediately.
1394	* Otherwise we use kcopyd to zero the data first.
1395	*/
1396	if (pool->pf.zero_new_blocks) {
1397	if (io_overwrites_block(pool, bio))
1398	remap_and_issue_overwrite(tc, bio, data_begin: data_block, m);
1399	else {
1400	ll_zero(tc, m, begin: data_block * pool->sectors_per_block,
1401	end: (data_block + 1) * pool->sectors_per_block);
1402	}
1403	} else
1404	process_prepared_mapping(m);
1405	}
1406
1407	static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
1408	dm_block_t data_dest,
1409	struct dm_bio_prison_cell *cell, struct bio *bio)
1410	{
1411	struct pool *pool = tc->pool;
1412	sector_t virt_block_begin = virt_block * pool->sectors_per_block;
1413	sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
1414
1415	if (virt_block_end <= tc->origin_size) {
1416	schedule_copy(tc, virt_block, origin: tc->origin_dev,
1417	data_origin: virt_block, data_dest, cell, bio,
1418	len: pool->sectors_per_block);
1419
1420	} else if (virt_block_begin < tc->origin_size) {
1421	schedule_copy(tc, virt_block, origin: tc->origin_dev,
1422	data_origin: virt_block, data_dest, cell, bio,
1423	len: tc->origin_size - virt_block_begin);
1424
1425	} else
1426	schedule_zero(tc, virt_block, data_block: data_dest, cell, bio);
1427	}
1428
1429	static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1430
1431	static void requeue_bios(struct pool *pool);
1432
1433	static bool is_read_only_pool_mode(enum pool_mode mode)
1434	{
1435	return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY);
1436	}
1437
1438	static bool is_read_only(struct pool *pool)
1439	{
1440	return is_read_only_pool_mode(mode: get_pool_mode(pool));
1441	}
1442
1443	static void check_for_metadata_space(struct pool *pool)
1444	{
1445	int r;
1446	const char *ooms_reason = NULL;
1447	dm_block_t nr_free;
1448
1449	r = dm_pool_get_free_metadata_block_count(pmd: pool->pmd, result: &nr_free);
1450	if (r)
1451	ooms_reason = "Could not get free metadata blocks";
1452	else if (!nr_free)
1453	ooms_reason = "No free metadata blocks";
1454
1455	if (ooms_reason && !is_read_only(pool)) {
1456	DMERR("%s", ooms_reason);
1457	set_pool_mode(pool, new_mode: PM_OUT_OF_METADATA_SPACE);
1458	}
1459	}
1460
1461	static void check_for_data_space(struct pool *pool)
1462	{
1463	int r;
1464	dm_block_t nr_free;
1465
1466	if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
1467	return;
1468
1469	r = dm_pool_get_free_block_count(pmd: pool->pmd, result: &nr_free);
1470	if (r)
1471	return;
1472
1473	if (nr_free) {
1474	set_pool_mode(pool, new_mode: PM_WRITE);
1475	requeue_bios(pool);
1476	}
1477	}
1478
1479	/*
1480	* A non-zero return indicates read_only or fail_io mode.
1481	* Many callers don't care about the return value.
1482	*/
1483	static int commit(struct pool *pool)
1484	{
1485	int r;
1486
1487	if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE)
1488	return -EINVAL;
1489
1490	r = dm_pool_commit_metadata(pmd: pool->pmd);
1491	if (r)
1492	metadata_operation_failed(pool, op: "dm_pool_commit_metadata", r);
1493	else {
1494	check_for_metadata_space(pool);
1495	check_for_data_space(pool);
1496	}
1497
1498	return r;
1499	}
1500
1501	static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
1502	{
1503	if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1504	DMWARN("%s: reached low water mark for data device: sending event.",
1505	dm_device_name(pool->pool_md));
1506	spin_lock_irq(lock: &pool->lock);
1507	pool->low_water_triggered = true;
1508	spin_unlock_irq(lock: &pool->lock);
1509	dm_table_event(t: pool->ti->table);
1510	}
1511	}
1512
1513	static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1514	{
1515	int r;
1516	dm_block_t free_blocks;
1517	struct pool *pool = tc->pool;
1518
1519	if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
1520	return -EINVAL;
1521
1522	r = dm_pool_get_free_block_count(pmd: pool->pmd, result: &free_blocks);
1523	if (r) {
1524	metadata_operation_failed(pool, op: "dm_pool_get_free_block_count", r);
1525	return r;
1526	}
1527
1528	check_low_water_mark(pool, free_blocks);
1529
1530	if (!free_blocks) {
1531	/*
1532	* Try to commit to see if that will free up some
1533	* more space.
1534	*/
1535	r = commit(pool);
1536	if (r)
1537	return r;
1538
1539	r = dm_pool_get_free_block_count(pmd: pool->pmd, result: &free_blocks);
1540	if (r) {
1541	metadata_operation_failed(pool, op: "dm_pool_get_free_block_count", r);
1542	return r;
1543	}
1544
1545	if (!free_blocks) {
1546	set_pool_mode(pool, new_mode: PM_OUT_OF_DATA_SPACE);
1547	return -ENOSPC;
1548	}
1549	}
1550
1551	r = dm_pool_alloc_data_block(pmd: pool->pmd, result);
1552	if (r) {
1553	if (r == -ENOSPC)
1554	set_pool_mode(pool, new_mode: PM_OUT_OF_DATA_SPACE);
1555	else
1556	metadata_operation_failed(pool, op: "dm_pool_alloc_data_block", r);
1557	return r;
1558	}
1559
1560	r = dm_pool_get_free_metadata_block_count(pmd: pool->pmd, result: &free_blocks);
1561	if (r) {
1562	metadata_operation_failed(pool, op: "dm_pool_get_free_metadata_block_count", r);
1563	return r;
1564	}
1565
1566	if (!free_blocks) {
1567	/* Let's commit before we use up the metadata reserve. */
1568	r = commit(pool);
1569	if (r)
1570	return r;
1571	}
1572
1573	return 0;
1574	}
1575
1576	/*
1577	* If we have run out of space, queue bios until the device is
1578	* resumed, presumably after having been reloaded with more space.
1579	*/
1580	static void retry_on_resume(struct bio *bio)
1581	{
1582	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
1583	struct thin_c *tc = h->tc;
1584
1585	spin_lock_irq(lock: &tc->lock);
1586	bio_list_add(bl: &tc->retry_on_resume_list, bio);
1587	spin_unlock_irq(lock: &tc->lock);
1588	}
1589
1590	static blk_status_t should_error_unserviceable_bio(struct pool *pool)
1591	{
1592	enum pool_mode m = get_pool_mode(pool);
1593
1594	switch (m) {
1595	case PM_WRITE:
1596	/* Shouldn't get here */
1597	DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
1598	return BLK_STS_IOERR;
1599
1600	case PM_OUT_OF_DATA_SPACE:
1601	return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0;
1602
1603	case PM_OUT_OF_METADATA_SPACE:
1604	case PM_READ_ONLY:
1605	case PM_FAIL:
1606	return BLK_STS_IOERR;
1607	default:
1608	/* Shouldn't get here */
1609	DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
1610	return BLK_STS_IOERR;
1611	}
1612	}
1613
1614	static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1615	{
1616	blk_status_t error = should_error_unserviceable_bio(pool);
1617
1618	if (error) {
1619	bio->bi_status = error;
1620	bio_endio(bio);
1621	} else
1622	retry_on_resume(bio);
1623	}
1624
1625	static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
1626	{
1627	struct bio *bio;
1628	struct bio_list bios;
1629	blk_status_t error;
1630
1631	error = should_error_unserviceable_bio(pool);
1632	if (error) {
1633	cell_error_with_code(pool, cell, error_code: error);
1634	return;
1635	}
1636
1637	bio_list_init(bl: &bios);
1638	cell_release(pool, cell, bios: &bios);
1639
1640	while ((bio = bio_list_pop(bl: &bios)))
1641	retry_on_resume(bio);
1642	}
1643
1644	static void process_discard_cell_no_passdown(struct thin_c *tc,
1645	struct dm_bio_prison_cell *virt_cell)
1646	{
1647	struct pool *pool = tc->pool;
1648	struct dm_thin_new_mapping *m = get_next_mapping(pool);
1649
1650	/*
1651	* We don't need to lock the data blocks, since there's no
1652	* passdown. We only lock data blocks for allocation and breaking sharing.
1653	*/
1654	m->tc = tc;
1655	m->virt_begin = virt_cell->key.block_begin;
1656	m->virt_end = virt_cell->key.block_end;
1657	m->cell = virt_cell;
1658	m->bio = virt_cell->holder;
1659
1660	if (!dm_deferred_set_add_work(ds: pool->all_io_ds, work: &m->list))
1661	pool->process_prepared_discard(m);
1662	}
1663
1664	static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
1665	struct bio *bio)
1666	{
1667	struct pool *pool = tc->pool;
1668
1669	int r;
1670	bool maybe_shared;
1671	struct dm_cell_key data_key;
1672	struct dm_bio_prison_cell *data_cell;
1673	struct dm_thin_new_mapping *m;
1674	dm_block_t virt_begin, virt_end, data_begin, data_end;
1675	dm_block_t len, next_boundary;
1676
1677	while (begin != end) {
1678	r = dm_thin_find_mapped_range(td: tc->td, begin, end, thin_begin: &virt_begin, thin_end: &virt_end,
1679	pool_begin: &data_begin, maybe_shared: &maybe_shared);
1680	if (r) {
1681	/*
1682	* Silently fail, letting any mappings we've
1683	* created complete.
1684	*/
1685	break;
1686	}
1687
1688	data_end = data_begin + (virt_end - virt_begin);
1689
1690	/*
1691	* Make sure the data region obeys the bio prison restrictions.
1692	*/
1693	while (data_begin < data_end) {
1694	r = ensure_next_mapping(pool);
1695	if (r)
1696	return; /* we did our best */
1697
1698	next_boundary = ((data_begin >> BIO_PRISON_MAX_RANGE_SHIFT) + 1)
1699	<< BIO_PRISON_MAX_RANGE_SHIFT;
1700	len = min_t(sector_t, data_end - data_begin, next_boundary - data_begin);
1701
1702	/* This key is certainly within range given the above splitting */
1703	(void) build_key(td: tc->td, ls: PHYSICAL, b: data_begin, e: data_begin + len, key: &data_key);
1704	if (bio_detain(pool: tc->pool, key: &data_key, NULL, cell_result: &data_cell)) {
1705	/* contention, we'll give up with this range */
1706	data_begin += len;
1707	continue;
1708	}
1709
1710	/*
1711	* IO may still be going to the destination block. We must
1712	* quiesce before we can do the removal.
1713	*/
1714	m = get_next_mapping(pool);
1715	m->tc = tc;
1716	m->maybe_shared = maybe_shared;
1717	m->virt_begin = virt_begin;
1718	m->virt_end = virt_begin + len;
1719	m->data_block = data_begin;
1720	m->cell = data_cell;
1721	m->bio = bio;
1722
1723	/*
1724	* The parent bio must not complete before sub discard bios are
1725	* chained to it (see end_discard's bio_chain)!
1726	*
1727	* This per-mapping bi_remaining increment is paired with
1728	* the implicit decrement that occurs via bio_endio() in
1729	* end_discard().
1730	*/
1731	bio_inc_remaining(bio);
1732	if (!dm_deferred_set_add_work(ds: pool->all_io_ds, work: &m->list))
1733	pool->process_prepared_discard(m);
1734
1735	virt_begin += len;
1736	data_begin += len;
1737	}
1738
1739	begin = virt_end;
1740	}
1741	}
1742
1743	static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
1744	{
1745	struct bio *bio = virt_cell->holder;
1746	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
1747
1748	/*
1749	* The virt_cell will only get freed once the origin bio completes.
1750	* This means it will remain locked while all the individual
1751	* passdown bios are in flight.
1752	*/
1753	h->cell = virt_cell;
1754	break_up_discard_bio(tc, begin: virt_cell->key.block_begin, end: virt_cell->key.block_end, bio);
1755
1756	/*
1757	* We complete the bio now, knowing that the bi_remaining field
1758	* will prevent completion until the sub range discards have
1759	* completed.
1760	*/
1761	bio_endio(bio);
1762	}
1763
1764	static void process_discard_bio(struct thin_c *tc, struct bio *bio)
1765	{
1766	dm_block_t begin, end;
1767	struct dm_cell_key virt_key;
1768	struct dm_bio_prison_cell *virt_cell;
1769
1770	get_bio_block_range(tc, bio, begin: &begin, end: &end);
1771	if (begin == end) {
1772	/*
1773	* The discard covers less than a block.
1774	*/
1775	bio_endio(bio);
1776	return;
1777	}
1778
1779	if (unlikely(!build_key(tc->td, VIRTUAL, begin, end, &virt_key))) {
1780	DMERR_LIMIT("Discard doesn't respect bio prison limits");
1781	bio_endio(bio);
1782	return;
1783	}
1784
1785	if (bio_detain(pool: tc->pool, key: &virt_key, bio, cell_result: &virt_cell)) {
1786	/*
1787	* Potential starvation issue: We're relying on the
1788	* fs/application being well behaved, and not trying to
1789	* send IO to a region at the same time as discarding it.
1790	* If they do this persistently then it's possible this
1791	* cell will never be granted.
1792	*/
1793	return;
1794	}
1795
1796	tc->pool->process_discard_cell(tc, virt_cell);
1797	}
1798
1799	static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1800	struct dm_cell_key *key,
1801	struct dm_thin_lookup_result *lookup_result,
1802	struct dm_bio_prison_cell *cell)
1803	{
1804	int r;
1805	dm_block_t data_block;
1806	struct pool *pool = tc->pool;
1807
1808	r = alloc_data_block(tc, result: &data_block);
1809	switch (r) {
1810	case 0:
1811	schedule_internal_copy(tc, virt_block: block, data_origin: lookup_result->block,
1812	data_dest: data_block, cell, bio);
1813	break;
1814
1815	case -ENOSPC:
1816	retry_bios_on_resume(pool, cell);
1817	break;
1818
1819	default:
1820	DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1821	__func__, r);
1822	cell_error(pool, cell);
1823	break;
1824	}
1825	}
1826
1827	static void __remap_and_issue_shared_cell(void *context,
1828	struct dm_bio_prison_cell *cell)
1829	{
1830	struct remap_info *info = context;
1831	struct bio *bio;
1832
1833	while ((bio = bio_list_pop(bl: &cell->bios))) {
1834	if (bio_data_dir(bio) == WRITE || op_is_flush(op: bio->bi_opf) ||
1835	bio_op(bio) == REQ_OP_DISCARD)
1836	bio_list_add(bl: &info->defer_bios, bio);
1837	else {
1838	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
1839
1840	h->shared_read_entry = dm_deferred_entry_inc(ds: info->tc->pool->shared_read_ds);
1841	inc_all_io_entry(pool: info->tc->pool, bio);
1842	bio_list_add(bl: &info->issue_bios, bio);
1843	}
1844	}
1845	}
1846
1847	static void remap_and_issue_shared_cell(struct thin_c *tc,
1848	struct dm_bio_prison_cell *cell,
1849	dm_block_t block)
1850	{
1851	struct bio *bio;
1852	struct remap_info info;
1853
1854	info.tc = tc;
1855	bio_list_init(bl: &info.defer_bios);
1856	bio_list_init(bl: &info.issue_bios);
1857
1858	cell_visit_release(pool: tc->pool, fn: __remap_and_issue_shared_cell,
1859	context: &info, cell);
1860
1861	while ((bio = bio_list_pop(bl: &info.defer_bios)))
1862	thin_defer_bio(tc, bio);
1863
1864	while ((bio = bio_list_pop(bl: &info.issue_bios)))
1865	remap_and_issue(tc, bio, block);
1866	}
1867
1868	static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1869	dm_block_t block,
1870	struct dm_thin_lookup_result *lookup_result,
1871	struct dm_bio_prison_cell *virt_cell)
1872	{
1873	struct dm_bio_prison_cell *data_cell;
1874	struct pool *pool = tc->pool;
1875	struct dm_cell_key key;
1876
1877	/*
1878	* If cell is already occupied, then sharing is already in the process
1879	* of being broken so we have nothing further to do here.
1880	*/
1881	build_data_key(td: tc->td, b: lookup_result->block, key: &key);
1882	if (bio_detain(pool, key: &key, bio, cell_result: &data_cell)) {
1883	cell_defer_no_holder(tc, cell: virt_cell);
1884	return;
1885	}
1886
1887	if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
1888	break_sharing(tc, bio, block, key: &key, lookup_result, cell: data_cell);
1889	cell_defer_no_holder(tc, cell: virt_cell);
1890	} else {
1891	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
1892
1893	h->shared_read_entry = dm_deferred_entry_inc(ds: pool->shared_read_ds);
1894	inc_all_io_entry(pool, bio);
1895	remap_and_issue(tc, bio, block: lookup_result->block);
1896
1897	remap_and_issue_shared_cell(tc, cell: data_cell, block: lookup_result->block);
1898	remap_and_issue_shared_cell(tc, cell: virt_cell, block: lookup_result->block);
1899	}
1900	}
1901
1902	static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
1903	struct dm_bio_prison_cell *cell)
1904	{
1905	int r;
1906	dm_block_t data_block;
1907	struct pool *pool = tc->pool;
1908
1909	/*
1910	* Remap empty bios (flushes) immediately, without provisioning.
1911	*/
1912	if (!bio->bi_iter.bi_size) {
1913	inc_all_io_entry(pool, bio);
1914	cell_defer_no_holder(tc, cell);
1915
1916	remap_and_issue(tc, bio, block: 0);
1917	return;
1918	}
1919
1920	/*
1921	* Fill read bios with zeroes and complete them immediately.
1922	*/
1923	if (bio_data_dir(bio) == READ) {
1924	zero_fill_bio(bio);
1925	cell_defer_no_holder(tc, cell);
1926	bio_endio(bio);
1927	return;
1928	}
1929
1930	r = alloc_data_block(tc, result: &data_block);
1931	switch (r) {
1932	case 0:
1933	if (tc->origin_dev)
1934	schedule_external_copy(tc, virt_block: block, data_dest: data_block, cell, bio);
1935	else
1936	schedule_zero(tc, virt_block: block, data_block, cell, bio);
1937	break;
1938
1939	case -ENOSPC:
1940	retry_bios_on_resume(pool, cell);
1941	break;
1942
1943	default:
1944	DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1945	__func__, r);
1946	cell_error(pool, cell);
1947	break;
1948	}
1949	}
1950
1951	static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1952	{
1953	int r;
1954	struct pool *pool = tc->pool;
1955	struct bio *bio = cell->holder;
1956	dm_block_t block = get_bio_block(tc, bio);
1957	struct dm_thin_lookup_result lookup_result;
1958
1959	if (tc->requeue_mode) {
1960	cell_requeue(pool, cell);
1961	return;
1962	}
1963
1964	r = dm_thin_find_block(td: tc->td, block, can_issue_io: 1, result: &lookup_result);
1965	switch (r) {
1966	case 0:
1967	if (lookup_result.shared)
1968	process_shared_bio(tc, bio, block, lookup_result: &lookup_result, virt_cell: cell);
1969	else {
1970	inc_all_io_entry(pool, bio);
1971	remap_and_issue(tc, bio, block: lookup_result.block);
1972	inc_remap_and_issue_cell(tc, cell, block: lookup_result.block);
1973	}
1974	break;
1975
1976	case -ENODATA:
1977	if (bio_data_dir(bio) == READ && tc->origin_dev) {
1978	inc_all_io_entry(pool, bio);
1979	cell_defer_no_holder(tc, cell);
1980
1981	if (bio_end_sector(bio) <= tc->origin_size)
1982	remap_to_origin_and_issue(tc, bio);
1983
1984	else if (bio->bi_iter.bi_sector < tc->origin_size) {
1985	zero_fill_bio(bio);
1986	bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
1987	remap_to_origin_and_issue(tc, bio);
1988
1989	} else {
1990	zero_fill_bio(bio);
1991	bio_endio(bio);
1992	}
1993	} else
1994	provision_block(tc, bio, block, cell);
1995	break;
1996
1997	default:
1998	DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1999	__func__, r);
2000	cell_defer_no_holder(tc, cell);
2001	bio_io_error(bio);
2002	break;
2003	}
2004	}
2005
2006	static void process_bio(struct thin_c *tc, struct bio *bio)
2007	{
2008	struct pool *pool = tc->pool;
2009	dm_block_t block = get_bio_block(tc, bio);
2010	struct dm_bio_prison_cell *cell;
2011	struct dm_cell_key key;
2012
2013	/*
2014	* If cell is already occupied, then the block is already
2015	* being provisioned so we have nothing further to do here.
2016	*/
2017	build_virtual_key(td: tc->td, b: block, key: &key);
2018	if (bio_detain(pool, key: &key, bio, cell_result: &cell))
2019	return;
2020
2021	process_cell(tc, cell);
2022	}
2023
2024	static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
2025	struct dm_bio_prison_cell *cell)
2026	{
2027	int r;
2028	int rw = bio_data_dir(bio);
2029	dm_block_t block = get_bio_block(tc, bio);
2030	struct dm_thin_lookup_result lookup_result;
2031
2032	r = dm_thin_find_block(td: tc->td, block, can_issue_io: 1, result: &lookup_result);
2033	switch (r) {
2034	case 0:
2035	if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
2036	handle_unserviceable_bio(pool: tc->pool, bio);
2037	if (cell)
2038	cell_defer_no_holder(tc, cell);
2039	} else {
2040	inc_all_io_entry(pool: tc->pool, bio);
2041	remap_and_issue(tc, bio, block: lookup_result.block);
2042	if (cell)
2043	inc_remap_and_issue_cell(tc, cell, block: lookup_result.block);
2044	}
2045	break;
2046
2047	case -ENODATA:
2048	if (cell)
2049	cell_defer_no_holder(tc, cell);
2050	if (rw != READ) {
2051	handle_unserviceable_bio(pool: tc->pool, bio);
2052	break;
2053	}
2054
2055	if (tc->origin_dev) {
2056	inc_all_io_entry(pool: tc->pool, bio);
2057	remap_to_origin_and_issue(tc, bio);
2058	break;
2059	}
2060
2061	zero_fill_bio(bio);
2062	bio_endio(bio);
2063	break;
2064
2065	default:
2066	DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
2067	__func__, r);
2068	if (cell)
2069	cell_defer_no_holder(tc, cell);
2070	bio_io_error(bio);
2071	break;
2072	}
2073	}
2074
2075	static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
2076	{
2077	__process_bio_read_only(tc, bio, NULL);
2078	}
2079
2080	static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2081	{
2082	__process_bio_read_only(tc, bio: cell->holder, cell);
2083	}
2084
2085	static void process_bio_success(struct thin_c *tc, struct bio *bio)
2086	{
2087	bio_endio(bio);
2088	}
2089
2090	static void process_bio_fail(struct thin_c *tc, struct bio *bio)
2091	{
2092	bio_io_error(bio);
2093	}
2094
2095	static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2096	{
2097	cell_success(pool: tc->pool, cell);
2098	}
2099
2100	static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2101	{
2102	cell_error(pool: tc->pool, cell);
2103	}
2104
2105	/*
2106	* FIXME: should we also commit due to size of transaction, measured in
2107	* metadata blocks?
2108	*/
2109	static int need_commit_due_to_time(struct pool *pool)
2110	{
2111	return !time_in_range(jiffies, pool->last_commit_jiffies,
2112	pool->last_commit_jiffies + COMMIT_PERIOD);
2113	}
2114
2115	#define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
2116	#define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
2117
2118	static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
2119	{
2120	struct rb_node **rbp, *parent;
2121	struct dm_thin_endio_hook *pbd;
2122	sector_t bi_sector = bio->bi_iter.bi_sector;
2123
2124	rbp = &tc->sort_bio_list.rb_node;
2125	parent = NULL;
2126	while (*rbp) {
2127	parent = *rbp;
2128	pbd = thin_pbd(parent);
2129
2130	if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
2131	rbp = &(*rbp)->rb_left;
2132	else
2133	rbp = &(*rbp)->rb_right;
2134	}
2135
2136	pbd = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
2137	rb_link_node(node: &pbd->rb_node, parent, rb_link: rbp);
2138	rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
2139	}
2140
2141	static void __extract_sorted_bios(struct thin_c *tc)
2142	{
2143	struct rb_node *node;
2144	struct dm_thin_endio_hook *pbd;
2145	struct bio *bio;
2146
2147	for (node = rb_first(root: &tc->sort_bio_list); node; node = rb_next(node)) {
2148	pbd = thin_pbd(node);
2149	bio = thin_bio(pbd);
2150
2151	bio_list_add(bl: &tc->deferred_bio_list, bio);
2152	rb_erase(&pbd->rb_node, &tc->sort_bio_list);
2153	}
2154
2155	WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
2156	}
2157
2158	static void __sort_thin_deferred_bios(struct thin_c *tc)
2159	{
2160	struct bio *bio;
2161	struct bio_list bios;
2162
2163	bio_list_init(bl: &bios);
2164	bio_list_merge(bl: &bios, bl2: &tc->deferred_bio_list);
2165	bio_list_init(bl: &tc->deferred_bio_list);
2166
2167	/* Sort deferred_bio_list using rb-tree */
2168	while ((bio = bio_list_pop(bl: &bios)))
2169	__thin_bio_rb_add(tc, bio);
2170
2171	/*
2172	* Transfer the sorted bios in sort_bio_list back to
2173	* deferred_bio_list to allow lockless submission of
2174	* all bios.
2175	*/
2176	__extract_sorted_bios(tc);
2177	}
2178
2179	static void process_thin_deferred_bios(struct thin_c *tc)
2180	{
2181	struct pool *pool = tc->pool;
2182	struct bio *bio;
2183	struct bio_list bios;
2184	struct blk_plug plug;
2185	unsigned int count = 0;
2186
2187	if (tc->requeue_mode) {
2188	error_thin_bio_list(tc, master: &tc->deferred_bio_list,
2189	BLK_STS_DM_REQUEUE);
2190	return;
2191	}
2192
2193	bio_list_init(bl: &bios);
2194
2195	spin_lock_irq(lock: &tc->lock);
2196
2197	if (bio_list_empty(bl: &tc->deferred_bio_list)) {
2198	spin_unlock_irq(lock: &tc->lock);
2199	return;
2200	}
2201
2202	__sort_thin_deferred_bios(tc);
2203
2204	bio_list_merge(bl: &bios, bl2: &tc->deferred_bio_list);
2205	bio_list_init(bl: &tc->deferred_bio_list);
2206
2207	spin_unlock_irq(lock: &tc->lock);
2208
2209	blk_start_plug(&plug);
2210	while ((bio = bio_list_pop(bl: &bios))) {
2211	/*
2212	* If we've got no free new_mapping structs, and processing
2213	* this bio might require one, we pause until there are some
2214	* prepared mappings to process.
2215	*/
2216	if (ensure_next_mapping(pool)) {
2217	spin_lock_irq(lock: &tc->lock);
2218	bio_list_add(bl: &tc->deferred_bio_list, bio);
2219	bio_list_merge(bl: &tc->deferred_bio_list, bl2: &bios);
2220	spin_unlock_irq(lock: &tc->lock);
2221	break;
2222	}
2223
2224	if (bio_op(bio) == REQ_OP_DISCARD)
2225	pool->process_discard(tc, bio);
2226	else
2227	pool->process_bio(tc, bio);
2228
2229	if ((count++ & 127) == 0) {
2230	throttle_work_update(t: &pool->throttle);
2231	dm_pool_issue_prefetches(pmd: pool->pmd);
2232	}
2233	cond_resched();
2234	}
2235	blk_finish_plug(&plug);
2236	}
2237
2238	static int cmp_cells(const void *lhs, const void *rhs)
2239	{
2240	struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);
2241	struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);
2242
2243	BUG_ON(!lhs_cell->holder);
2244	BUG_ON(!rhs_cell->holder);
2245
2246	if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
2247	return -1;
2248
2249	if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
2250	return 1;
2251
2252	return 0;
2253	}
2254
2255	static unsigned int sort_cells(struct pool *pool, struct list_head *cells)
2256	{
2257	unsigned int count = 0;
2258	struct dm_bio_prison_cell *cell, *tmp;
2259
2260	list_for_each_entry_safe(cell, tmp, cells, user_list) {
2261	if (count >= CELL_SORT_ARRAY_SIZE)
2262	break;
2263
2264	pool->cell_sort_array[count++] = cell;
2265	list_del(entry: &cell->user_list);
2266	}
2267
2268	sort(base: pool->cell_sort_array, num: count, size: sizeof(cell), cmp_func: cmp_cells, NULL);
2269
2270	return count;
2271	}
2272
2273	static void process_thin_deferred_cells(struct thin_c *tc)
2274	{
2275	struct pool *pool = tc->pool;
2276	struct list_head cells;
2277	struct dm_bio_prison_cell *cell;
2278	unsigned int i, j, count;
2279
2280	INIT_LIST_HEAD(list: &cells);
2281
2282	spin_lock_irq(lock: &tc->lock);
2283	list_splice_init(list: &tc->deferred_cells, head: &cells);
2284	spin_unlock_irq(lock: &tc->lock);
2285
2286	if (list_empty(head: &cells))
2287	return;
2288
2289	do {
2290	count = sort_cells(pool: tc->pool, cells: &cells);
2291
2292	for (i = 0; i < count; i++) {
2293	cell = pool->cell_sort_array[i];
2294	BUG_ON(!cell->holder);
2295
2296	/*
2297	* If we've got no free new_mapping structs, and processing
2298	* this bio might require one, we pause until there are some
2299	* prepared mappings to process.
2300	*/
2301	if (ensure_next_mapping(pool)) {
2302	for (j = i; j < count; j++)
2303	list_add(new: &pool->cell_sort_array[j]->user_list, head: &cells);
2304
2305	spin_lock_irq(lock: &tc->lock);
2306	list_splice(list: &cells, head: &tc->deferred_cells);
2307	spin_unlock_irq(lock: &tc->lock);
2308	return;
2309	}
2310
2311	if (bio_op(bio: cell->holder) == REQ_OP_DISCARD)
2312	pool->process_discard_cell(tc, cell);
2313	else
2314	pool->process_cell(tc, cell);
2315	}
2316	cond_resched();
2317	} while (!list_empty(head: &cells));
2318	}
2319
2320	static void thin_get(struct thin_c *tc);
2321	static void thin_put(struct thin_c *tc);
2322
2323	/*
2324	* We can't hold rcu_read_lock() around code that can block. So we
2325	* find a thin with the rcu lock held; bump a refcount; then drop
2326	* the lock.
2327	*/
2328	static struct thin_c *get_first_thin(struct pool *pool)
2329	{
2330	struct thin_c *tc = NULL;
2331
2332	rcu_read_lock();
2333	tc = list_first_or_null_rcu(&pool->active_thins, struct thin_c, list);
2334	if (tc)
2335	thin_get(tc);
2336	rcu_read_unlock();
2337
2338	return tc;
2339	}
2340
2341	static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
2342	{
2343	struct thin_c *old_tc = tc;
2344
2345	rcu_read_lock();
2346	list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
2347	thin_get(tc);
2348	thin_put(tc: old_tc);
2349	rcu_read_unlock();
2350	return tc;
2351	}
2352	thin_put(tc: old_tc);
2353	rcu_read_unlock();
2354
2355	return NULL;
2356	}
2357
2358	static void process_deferred_bios(struct pool *pool)
2359	{
2360	struct bio *bio;
2361	struct bio_list bios, bio_completions;
2362	struct thin_c *tc;
2363
2364	tc = get_first_thin(pool);
2365	while (tc) {
2366	process_thin_deferred_cells(tc);
2367	process_thin_deferred_bios(tc);
2368	tc = get_next_thin(pool, tc);
2369	}
2370
2371	/*
2372	* If there are any deferred flush bios, we must commit the metadata
2373	* before issuing them or signaling their completion.
2374	*/
2375	bio_list_init(bl: &bios);
2376	bio_list_init(bl: &bio_completions);
2377
2378	spin_lock_irq(lock: &pool->lock);
2379	bio_list_merge(bl: &bios, bl2: &pool->deferred_flush_bios);
2380	bio_list_init(bl: &pool->deferred_flush_bios);
2381
2382	bio_list_merge(bl: &bio_completions, bl2: &pool->deferred_flush_completions);
2383	bio_list_init(bl: &pool->deferred_flush_completions);
2384	spin_unlock_irq(lock: &pool->lock);
2385
2386	if (bio_list_empty(bl: &bios) && bio_list_empty(bl: &bio_completions) &&
2387	!(dm_pool_changed_this_transaction(pmd: pool->pmd) && need_commit_due_to_time(pool)))
2388	return;
2389
2390	if (commit(pool)) {
2391	bio_list_merge(bl: &bios, bl2: &bio_completions);
2392
2393	while ((bio = bio_list_pop(bl: &bios)))
2394	bio_io_error(bio);
2395	return;
2396	}
2397	pool->last_commit_jiffies = jiffies;
2398
2399	while ((bio = bio_list_pop(bl: &bio_completions)))
2400	bio_endio(bio);
2401
2402	while ((bio = bio_list_pop(bl: &bios))) {
2403	/*
2404	* The data device was flushed as part of metadata commit,
2405	* so complete redundant flushes immediately.
2406	*/
2407	if (bio->bi_opf & REQ_PREFLUSH)
2408	bio_endio(bio);
2409	else
2410	dm_submit_bio_remap(clone: bio, NULL);
2411	}
2412	}
2413
2414	static void do_worker(struct work_struct *ws)
2415	{
2416	struct pool *pool = container_of(ws, struct pool, worker);
2417
2418	throttle_work_start(t: &pool->throttle);
2419	dm_pool_issue_prefetches(pmd: pool->pmd);
2420	throttle_work_update(t: &pool->throttle);
2421	process_prepared(pool, head: &pool->prepared_mappings, fn: &pool->process_prepared_mapping);
2422	throttle_work_update(t: &pool->throttle);
2423	process_prepared(pool, head: &pool->prepared_discards, fn: &pool->process_prepared_discard);
2424	throttle_work_update(t: &pool->throttle);
2425	process_prepared(pool, head: &pool->prepared_discards_pt2, fn: &pool->process_prepared_discard_pt2);
2426	throttle_work_update(t: &pool->throttle);
2427	process_deferred_bios(pool);
2428	throttle_work_complete(t: &pool->throttle);
2429	}
2430
2431	/*
2432	* We want to commit periodically so that not too much
2433	* unwritten data builds up.
2434	*/
2435	static void do_waker(struct work_struct *ws)
2436	{
2437	struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
2438
2439	wake_worker(pool);
2440	queue_delayed_work(wq: pool->wq, dwork: &pool->waker, COMMIT_PERIOD);
2441	}
2442
2443	/*
2444	* We're holding onto IO to allow userland time to react. After the
2445	* timeout either the pool will have been resized (and thus back in
2446	* PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
2447	*/
2448	static void do_no_space_timeout(struct work_struct *ws)
2449	{
2450	struct pool *pool = container_of(to_delayed_work(ws), struct pool,
2451	no_space_timeout);
2452
2453	if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {
2454	pool->pf.error_if_no_space = true;
2455	notify_of_pool_mode_change(pool);
2456	error_retry_list_with_code(pool, BLK_STS_NOSPC);
2457	}
2458	}
2459
2460	/*----------------------------------------------------------------*/
2461
2462	struct pool_work {
2463	struct work_struct worker;
2464	struct completion complete;
2465	};
2466
2467	static struct pool_work *to_pool_work(struct work_struct *ws)
2468	{
2469	return container_of(ws, struct pool_work, worker);
2470	}
2471
2472	static void pool_work_complete(struct pool_work *pw)
2473	{
2474	complete(&pw->complete);
2475	}
2476
2477	static void pool_work_wait(struct pool_work *pw, struct pool *pool,
2478	void (*fn)(struct work_struct *))
2479	{
2480	INIT_WORK_ONSTACK(&pw->worker, fn);
2481	init_completion(x: &pw->complete);
2482	queue_work(wq: pool->wq, work: &pw->worker);
2483	wait_for_completion(&pw->complete);
2484	destroy_work_on_stack(work: &pw->worker);
2485	}
2486
2487	/*----------------------------------------------------------------*/
2488
2489	struct noflush_work {
2490	struct pool_work pw;
2491	struct thin_c *tc;
2492	};
2493
2494	static struct noflush_work *to_noflush(struct work_struct *ws)
2495	{
2496	return container_of(to_pool_work(ws), struct noflush_work, pw);
2497	}
2498
2499	static void do_noflush_start(struct work_struct *ws)
2500	{
2501	struct noflush_work *w = to_noflush(ws);
2502
2503	w->tc->requeue_mode = true;
2504	requeue_io(tc: w->tc);
2505	pool_work_complete(pw: &w->pw);
2506	}
2507
2508	static void do_noflush_stop(struct work_struct *ws)
2509	{
2510	struct noflush_work *w = to_noflush(ws);
2511
2512	w->tc->requeue_mode = false;
2513	pool_work_complete(pw: &w->pw);
2514	}
2515
2516	static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
2517	{
2518	struct noflush_work w;
2519
2520	w.tc = tc;
2521	pool_work_wait(pw: &w.pw, pool: tc->pool, fn);
2522	}
2523
2524	/*----------------------------------------------------------------*/
2525
2526	static void set_discard_callbacks(struct pool *pool)
2527	{
2528	struct pool_c *pt = pool->ti->private;
2529
2530	if (pt->adjusted_pf.discard_passdown) {
2531	pool->process_discard_cell = process_discard_cell_passdown;
2532	pool->process_prepared_discard = process_prepared_discard_passdown_pt1;
2533	pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2;
2534	} else {
2535	pool->process_discard_cell = process_discard_cell_no_passdown;
2536	pool->process_prepared_discard = process_prepared_discard_no_passdown;
2537	}
2538	}
2539
2540	static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
2541	{
2542	struct pool_c *pt = pool->ti->private;
2543	bool needs_check = dm_pool_metadata_needs_check(pmd: pool->pmd);
2544	enum pool_mode old_mode = get_pool_mode(pool);
2545	unsigned long no_space_timeout = READ_ONCE(no_space_timeout_secs) * HZ;
2546
2547	/*
2548	* Never allow the pool to transition to PM_WRITE mode if user
2549	* intervention is required to verify metadata and data consistency.
2550	*/
2551	if (new_mode == PM_WRITE && needs_check) {
2552	DMERR("%s: unable to switch pool to write mode until repaired.",
2553	dm_device_name(pool->pool_md));
2554	if (old_mode != new_mode)
2555	new_mode = old_mode;
2556	else
2557	new_mode = PM_READ_ONLY;
2558	}
2559	/*
2560	* If we were in PM_FAIL mode, rollback of metadata failed. We're
2561	* not going to recover without a thin_repair. So we never let the
2562	* pool move out of the old mode.
2563	*/
2564	if (old_mode == PM_FAIL)
2565	new_mode = old_mode;
2566
2567	switch (new_mode) {
2568	case PM_FAIL:
2569	dm_pool_metadata_read_only(pmd: pool->pmd);
2570	pool->process_bio = process_bio_fail;
2571	pool->process_discard = process_bio_fail;
2572	pool->process_cell = process_cell_fail;
2573	pool->process_discard_cell = process_cell_fail;
2574	pool->process_prepared_mapping = process_prepared_mapping_fail;
2575	pool->process_prepared_discard = process_prepared_discard_fail;
2576
2577	error_retry_list(pool);
2578	break;
2579
2580	case PM_OUT_OF_METADATA_SPACE:
2581	case PM_READ_ONLY:
2582	dm_pool_metadata_read_only(pmd: pool->pmd);
2583	pool->process_bio = process_bio_read_only;
2584	pool->process_discard = process_bio_success;
2585	pool->process_cell = process_cell_read_only;
2586	pool->process_discard_cell = process_cell_success;
2587	pool->process_prepared_mapping = process_prepared_mapping_fail;
2588	pool->process_prepared_discard = process_prepared_discard_success;
2589
2590	error_retry_list(pool);
2591	break;
2592
2593	case PM_OUT_OF_DATA_SPACE:
2594	/*
2595	* Ideally we'd never hit this state; the low water mark
2596	* would trigger userland to extend the pool before we
2597	* completely run out of data space. However, many small
2598	* IOs to unprovisioned space can consume data space at an
2599	* alarming rate. Adjust your low water mark if you're
2600	* frequently seeing this mode.
2601	*/
2602	pool->out_of_data_space = true;
2603	pool->process_bio = process_bio_read_only;
2604	pool->process_discard = process_discard_bio;
2605	pool->process_cell = process_cell_read_only;
2606	pool->process_prepared_mapping = process_prepared_mapping;
2607	set_discard_callbacks(pool);
2608
2609	if (!pool->pf.error_if_no_space && no_space_timeout)
2610	queue_delayed_work(wq: pool->wq, dwork: &pool->no_space_timeout, delay: no_space_timeout);
2611	break;
2612
2613	case PM_WRITE:
2614	if (old_mode == PM_OUT_OF_DATA_SPACE)
2615	cancel_delayed_work_sync(dwork: &pool->no_space_timeout);
2616	pool->out_of_data_space = false;
2617	pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space;
2618	dm_pool_metadata_read_write(pmd: pool->pmd);
2619	pool->process_bio = process_bio;
2620	pool->process_discard = process_discard_bio;
2621	pool->process_cell = process_cell;
2622	pool->process_prepared_mapping = process_prepared_mapping;
2623	set_discard_callbacks(pool);
2624	break;
2625	}
2626
2627	pool->pf.mode = new_mode;
2628	/*
2629	* The pool mode may have changed, sync it so bind_control_target()
2630	* doesn't cause an unexpected mode transition on resume.
2631	*/
2632	pt->adjusted_pf.mode = new_mode;
2633
2634	if (old_mode != new_mode)
2635	notify_of_pool_mode_change(pool);
2636	}
2637
2638	static void abort_transaction(struct pool *pool)
2639	{
2640	const char *dev_name = dm_device_name(md: pool->pool_md);
2641
2642	DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
2643	if (dm_pool_abort_metadata(pmd: pool->pmd)) {
2644	DMERR("%s: failed to abort metadata transaction", dev_name);
2645	set_pool_mode(pool, new_mode: PM_FAIL);
2646	}
2647
2648	if (dm_pool_metadata_set_needs_check(pmd: pool->pmd)) {
2649	DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
2650	set_pool_mode(pool, new_mode: PM_FAIL);
2651	}
2652	}
2653
2654	static void metadata_operation_failed(struct pool *pool, const char *op, int r)
2655	{
2656	DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
2657	dm_device_name(pool->pool_md), op, r);
2658
2659	abort_transaction(pool);
2660	set_pool_mode(pool, new_mode: PM_READ_ONLY);
2661	}
2662
2663	/*----------------------------------------------------------------*/
2664
2665	/*
2666	* Mapping functions.
2667	*/
2668
2669	/*
2670	* Called only while mapping a thin bio to hand it over to the workqueue.
2671	*/
2672	static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
2673	{
2674	struct pool *pool = tc->pool;
2675
2676	spin_lock_irq(lock: &tc->lock);
2677	bio_list_add(bl: &tc->deferred_bio_list, bio);
2678	spin_unlock_irq(lock: &tc->lock);
2679
2680	wake_worker(pool);
2681	}
2682
2683	static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)
2684	{
2685	struct pool *pool = tc->pool;
2686
2687	throttle_lock(t: &pool->throttle);
2688	thin_defer_bio(tc, bio);
2689	throttle_unlock(t: &pool->throttle);
2690	}
2691
2692	static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2693	{
2694	struct pool *pool = tc->pool;
2695
2696	throttle_lock(t: &pool->throttle);
2697	spin_lock_irq(lock: &tc->lock);
2698	list_add_tail(new: &cell->user_list, head: &tc->deferred_cells);
2699	spin_unlock_irq(lock: &tc->lock);
2700	throttle_unlock(t: &pool->throttle);
2701
2702	wake_worker(pool);
2703	}
2704
2705	static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
2706	{
2707	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
2708
2709	h->tc = tc;
2710	h->shared_read_entry = NULL;
2711	h->all_io_entry = NULL;
2712	h->overwrite_mapping = NULL;
2713	h->cell = NULL;
2714	}
2715
2716	/*
2717	* Non-blocking function called from the thin target's map function.
2718	*/
2719	static int thin_bio_map(struct dm_target *ti, struct bio *bio)
2720	{
2721	int r;
2722	struct thin_c *tc = ti->private;
2723	dm_block_t block = get_bio_block(tc, bio);
2724	struct dm_thin_device *td = tc->td;
2725	struct dm_thin_lookup_result result;
2726	struct dm_bio_prison_cell *virt_cell, *data_cell;
2727	struct dm_cell_key key;
2728
2729	thin_hook_bio(tc, bio);
2730
2731	if (tc->requeue_mode) {
2732	bio->bi_status = BLK_STS_DM_REQUEUE;
2733	bio_endio(bio);
2734	return DM_MAPIO_SUBMITTED;
2735	}
2736
2737	if (get_pool_mode(pool: tc->pool) == PM_FAIL) {
2738	bio_io_error(bio);
2739	return DM_MAPIO_SUBMITTED;
2740	}
2741
2742	if (op_is_flush(op: bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) {
2743	thin_defer_bio_with_throttle(tc, bio);
2744	return DM_MAPIO_SUBMITTED;
2745	}
2746
2747	/*
2748	* We must hold the virtual cell before doing the lookup, otherwise
2749	* there's a race with discard.
2750	*/
2751	build_virtual_key(td: tc->td, b: block, key: &key);
2752	if (bio_detain(pool: tc->pool, key: &key, bio, cell_result: &virt_cell))
2753	return DM_MAPIO_SUBMITTED;
2754
2755	r = dm_thin_find_block(td, block, can_issue_io: 0, result: &result);
2756
2757	/*
2758	* Note that we defer readahead too.
2759	*/
2760	switch (r) {
2761	case 0:
2762	if (unlikely(result.shared)) {
2763	/*
2764	* We have a race condition here between the
2765	* result.shared value returned by the lookup and
2766	* snapshot creation, which may cause new
2767	* sharing.
2768	*
2769	* To avoid this always quiesce the origin before
2770	* taking the snap. You want to do this anyway to
2771	* ensure a consistent application view
2772	* (i.e. lockfs).
2773	*
2774	* More distant ancestors are irrelevant. The
2775	* shared flag will be set in their case.
2776	*/
2777	thin_defer_cell(tc, cell: virt_cell);
2778	return DM_MAPIO_SUBMITTED;
2779	}
2780
2781	build_data_key(td: tc->td, b: result.block, key: &key);
2782	if (bio_detain(pool: tc->pool, key: &key, bio, cell_result: &data_cell)) {
2783	cell_defer_no_holder(tc, cell: virt_cell);
2784	return DM_MAPIO_SUBMITTED;
2785	}
2786
2787	inc_all_io_entry(pool: tc->pool, bio);
2788	cell_defer_no_holder(tc, cell: data_cell);
2789	cell_defer_no_holder(tc, cell: virt_cell);
2790
2791	remap(tc, bio, block: result.block);
2792	return DM_MAPIO_REMAPPED;
2793
2794	case -ENODATA:
2795	case -EWOULDBLOCK:
2796	thin_defer_cell(tc, cell: virt_cell);
2797	return DM_MAPIO_SUBMITTED;
2798
2799	default:
2800	/*
2801	* Must always call bio_io_error on failure.
2802	* dm_thin_find_block can fail with -EINVAL if the
2803	* pool is switched to fail-io mode.
2804	*/
2805	bio_io_error(bio);
2806	cell_defer_no_holder(tc, cell: virt_cell);
2807	return DM_MAPIO_SUBMITTED;
2808	}
2809	}
2810
2811	static void requeue_bios(struct pool *pool)
2812	{
2813	struct thin_c *tc;
2814
2815	rcu_read_lock();
2816	list_for_each_entry_rcu(tc, &pool->active_thins, list) {
2817	spin_lock_irq(lock: &tc->lock);
2818	bio_list_merge(bl: &tc->deferred_bio_list, bl2: &tc->retry_on_resume_list);
2819	bio_list_init(bl: &tc->retry_on_resume_list);
2820	spin_unlock_irq(lock: &tc->lock);
2821	}
2822	rcu_read_unlock();
2823	}
2824
2825	/*
2826	*--------------------------------------------------------------
2827	* Binding of control targets to a pool object
2828	*--------------------------------------------------------------
2829	*/
2830	static bool is_factor(sector_t block_size, uint32_t n)
2831	{
2832	return !sector_div(block_size, n);
2833	}
2834
2835	/*
2836	* If discard_passdown was enabled verify that the data device
2837	* supports discards. Disable discard_passdown if not.
2838	*/
2839	static void disable_discard_passdown_if_not_supported(struct pool_c *pt)
2840	{
2841	struct pool *pool = pt->pool;
2842	struct block_device *data_bdev = pt->data_dev->bdev;
2843	struct queue_limits *data_limits = bdev_limits(bdev: data_bdev);
2844	const char *reason = NULL;
2845
2846	if (!pt->adjusted_pf.discard_passdown)
2847	return;
2848
2849	if (!bdev_max_discard_sectors(bdev: pt->data_dev->bdev))
2850	reason = "discard unsupported";
2851
2852	else if (data_limits->max_discard_sectors < pool->sectors_per_block)
2853	reason = "max discard sectors smaller than a block";
2854
2855	if (reason) {
2856	DMWARN("Data device (%pg) %s: Disabling discard passdown.", data_bdev, reason);
2857	pt->adjusted_pf.discard_passdown = false;
2858	}
2859	}
2860
2861	static int bind_control_target(struct pool *pool, struct dm_target *ti)
2862	{
2863	struct pool_c *pt = ti->private;
2864
2865	/*
2866	* We want to make sure that a pool in PM_FAIL mode is never upgraded.
2867	*/
2868	enum pool_mode old_mode = get_pool_mode(pool);
2869	enum pool_mode new_mode = pt->adjusted_pf.mode;
2870
2871	/*
2872	* Don't change the pool's mode until set_pool_mode() below.
2873	* Otherwise the pool's process_* function pointers may
2874	* not match the desired pool mode.
2875	*/
2876	pt->adjusted_pf.mode = old_mode;
2877
2878	pool->ti = ti;
2879	pool->pf = pt->adjusted_pf;
2880	pool->low_water_blocks = pt->low_water_blocks;
2881
2882	set_pool_mode(pool, new_mode);
2883
2884	return 0;
2885	}
2886
2887	static void unbind_control_target(struct pool *pool, struct dm_target *ti)
2888	{
2889	if (pool->ti == ti)
2890	pool->ti = NULL;
2891	}
2892
2893	/*
2894	*--------------------------------------------------------------
2895	* Pool creation
2896	*--------------------------------------------------------------
2897	*/
2898	/* Initialize pool features. */
2899	static void pool_features_init(struct pool_features *pf)
2900	{
2901	pf->mode = PM_WRITE;
2902	pf->zero_new_blocks = true;
2903	pf->discard_enabled = true;
2904	pf->discard_passdown = true;
2905	pf->error_if_no_space = false;
2906	}
2907
2908	static void __pool_destroy(struct pool *pool)
2909	{
2910	__pool_table_remove(pool);
2911
2912	vfree(addr: pool->cell_sort_array);
2913	if (dm_pool_metadata_close(pmd: pool->pmd) < 0)
2914	DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2915
2916	dm_bio_prison_destroy(prison: pool->prison);
2917	dm_kcopyd_client_destroy(kc: pool->copier);
2918
2919	cancel_delayed_work_sync(dwork: &pool->waker);
2920	cancel_delayed_work_sync(dwork: &pool->no_space_timeout);
2921	if (pool->wq)
2922	destroy_workqueue(wq: pool->wq);
2923
2924	if (pool->next_mapping)
2925	mempool_free(element: pool->next_mapping, pool: &pool->mapping_pool);
2926	mempool_exit(pool: &pool->mapping_pool);
2927	dm_deferred_set_destroy(ds: pool->shared_read_ds);
2928	dm_deferred_set_destroy(ds: pool->all_io_ds);
2929	kfree(objp: pool);
2930	}
2931
2932	static struct kmem_cache *_new_mapping_cache;
2933
2934	static struct pool *pool_create(struct mapped_device *pool_md,
2935	struct block_device *metadata_dev,
2936	struct block_device *data_dev,
2937	unsigned long block_size,
2938	int read_only, char **error)
2939	{
2940	int r;
2941	void *err_p;
2942	struct pool *pool;
2943	struct dm_pool_metadata *pmd;
2944	bool format_device = read_only ? false : true;
2945
2946	pmd = dm_pool_metadata_open(bdev: metadata_dev, data_block_size: block_size, format_device);
2947	if (IS_ERR(ptr: pmd)) {
2948	*error = "Error creating metadata object";
2949	return ERR_CAST(ptr: pmd);
2950	}
2951
2952	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2953	if (!pool) {
2954	*error = "Error allocating memory for pool";
2955	err_p = ERR_PTR(error: -ENOMEM);
2956	goto bad_pool;
2957	}
2958
2959	pool->pmd = pmd;
2960	pool->sectors_per_block = block_size;
2961	if (block_size & (block_size - 1))
2962	pool->sectors_per_block_shift = -1;
2963	else
2964	pool->sectors_per_block_shift = __ffs(block_size);
2965	pool->low_water_blocks = 0;
2966	pool_features_init(pf: &pool->pf);
2967	pool->prison = dm_bio_prison_create();
2968	if (!pool->prison) {
2969	*error = "Error creating pool's bio prison";
2970	err_p = ERR_PTR(error: -ENOMEM);
2971	goto bad_prison;
2972	}
2973
2974	pool->copier = dm_kcopyd_client_create(throttle: &dm_kcopyd_throttle);
2975	if (IS_ERR(ptr: pool->copier)) {
2976	r = PTR_ERR(ptr: pool->copier);
2977	*error = "Error creating pool's kcopyd client";
2978	err_p = ERR_PTR(error: r);
2979	goto bad_kcopyd_client;
2980	}
2981
2982	/*
2983	* Create singlethreaded workqueue that will service all devices
2984	* that use this metadata.
2985	*/
2986	pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2987	if (!pool->wq) {
2988	*error = "Error creating pool's workqueue";
2989	err_p = ERR_PTR(error: -ENOMEM);
2990	goto bad_wq;
2991	}
2992
2993	throttle_init(t: &pool->throttle);
2994	INIT_WORK(&pool->worker, do_worker);
2995	INIT_DELAYED_WORK(&pool->waker, do_waker);
2996	INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
2997	spin_lock_init(&pool->lock);
2998	bio_list_init(bl: &pool->deferred_flush_bios);
2999	bio_list_init(bl: &pool->deferred_flush_completions);
3000	INIT_LIST_HEAD(list: &pool->prepared_mappings);
3001	INIT_LIST_HEAD(list: &pool->prepared_discards);
3002	INIT_LIST_HEAD(list: &pool->prepared_discards_pt2);
3003	INIT_LIST_HEAD(list: &pool->active_thins);
3004	pool->low_water_triggered = false;
3005	pool->suspended = true;
3006	pool->out_of_data_space = false;
3007
3008	pool->shared_read_ds = dm_deferred_set_create();
3009	if (!pool->shared_read_ds) {
3010	*error = "Error creating pool's shared read deferred set";
3011	err_p = ERR_PTR(error: -ENOMEM);
3012	goto bad_shared_read_ds;
3013	}
3014
3015	pool->all_io_ds = dm_deferred_set_create();
3016	if (!pool->all_io_ds) {
3017	*error = "Error creating pool's all io deferred set";
3018	err_p = ERR_PTR(error: -ENOMEM);
3019	goto bad_all_io_ds;
3020	}
3021
3022	pool->next_mapping = NULL;
3023	r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE,
3024	_new_mapping_cache);
3025	if (r) {
3026	*error = "Error creating pool's mapping mempool";
3027	err_p = ERR_PTR(error: r);
3028	goto bad_mapping_pool;
3029	}
3030
3031	pool->cell_sort_array =
3032	vmalloc_array(CELL_SORT_ARRAY_SIZE,
3033	sizeof(*pool->cell_sort_array));
3034	if (!pool->cell_sort_array) {
3035	*error = "Error allocating cell sort array";
3036	err_p = ERR_PTR(error: -ENOMEM);
3037	goto bad_sort_array;
3038	}
3039
3040	pool->ref_count = 1;
3041	pool->last_commit_jiffies = jiffies;
3042	pool->pool_md = pool_md;
3043	pool->md_dev = metadata_dev;
3044	pool->data_dev = data_dev;
3045	__pool_table_insert(pool);
3046
3047	return pool;
3048
3049	bad_sort_array:
3050	mempool_exit(pool: &pool->mapping_pool);
3051	bad_mapping_pool:
3052	dm_deferred_set_destroy(ds: pool->all_io_ds);
3053	bad_all_io_ds:
3054	dm_deferred_set_destroy(ds: pool->shared_read_ds);
3055	bad_shared_read_ds:
3056	destroy_workqueue(wq: pool->wq);
3057	bad_wq:
3058	dm_kcopyd_client_destroy(kc: pool->copier);
3059	bad_kcopyd_client:
3060	dm_bio_prison_destroy(prison: pool->prison);
3061	bad_prison:
3062	kfree(objp: pool);
3063	bad_pool:
3064	if (dm_pool_metadata_close(pmd))
3065	DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
3066
3067	return err_p;
3068	}
3069
3070	static void __pool_inc(struct pool *pool)
3071	{
3072	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
3073	pool->ref_count++;
3074	}
3075
3076	static void __pool_dec(struct pool *pool)
3077	{
3078	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
3079	BUG_ON(!pool->ref_count);
3080	if (!--pool->ref_count)
3081	__pool_destroy(pool);
3082	}
3083
3084	static struct pool *__pool_find(struct mapped_device *pool_md,
3085	struct block_device *metadata_dev,
3086	struct block_device *data_dev,
3087	unsigned long block_size, int read_only,
3088	char **error, int *created)
3089	{
3090	struct pool *pool = __pool_table_lookup_metadata_dev(md_dev: metadata_dev);
3091
3092	if (pool) {
3093	if (pool->pool_md != pool_md) {
3094	*error = "metadata device already in use by a pool";
3095	return ERR_PTR(error: -EBUSY);
3096	}
3097	if (pool->data_dev != data_dev) {
3098	*error = "data device already in use by a pool";
3099	return ERR_PTR(error: -EBUSY);
3100	}
3101	__pool_inc(pool);
3102
3103	} else {
3104	pool = __pool_table_lookup(md: pool_md);
3105	if (pool) {
3106	if (pool->md_dev != metadata_dev || pool->data_dev != data_dev) {
3107	*error = "different pool cannot replace a pool";
3108	return ERR_PTR(error: -EINVAL);
3109	}
3110	__pool_inc(pool);
3111
3112	} else {
3113	pool = pool_create(pool_md, metadata_dev, data_dev, block_size, read_only, error);
3114	*created = 1;
3115	}
3116	}
3117
3118	return pool;
3119	}
3120
3121	/*
3122	*--------------------------------------------------------------
3123	* Pool target methods
3124	*--------------------------------------------------------------
3125	*/
3126	static void pool_dtr(struct dm_target *ti)
3127	{
3128	struct pool_c *pt = ti->private;
3129
3130	mutex_lock(&dm_thin_pool_table.mutex);
3131
3132	unbind_control_target(pool: pt->pool, ti);
3133	__pool_dec(pool: pt->pool);
3134	dm_put_device(ti, d: pt->metadata_dev);
3135	dm_put_device(ti, d: pt->data_dev);
3136	kfree(objp: pt);
3137
3138	mutex_unlock(lock: &dm_thin_pool_table.mutex);
3139	}
3140
3141	static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
3142	struct dm_target *ti)
3143	{
3144	int r;
3145	unsigned int argc;
3146	const char *arg_name;
3147
3148	static const struct dm_arg _args[] = {
3149	{0, 4, "Invalid number of pool feature arguments"},
3150	};
3151
3152	/*
3153	* No feature arguments supplied.
3154	*/
3155	if (!as->argc)
3156	return 0;
3157
3158	r = dm_read_arg_group(arg: _args, arg_set: as, num_args: &argc, error: &ti->error);
3159	if (r)
3160	return -EINVAL;
3161
3162	while (argc && !r) {
3163	arg_name = dm_shift_arg(as);
3164	argc--;
3165
3166	if (!strcasecmp(s1: arg_name, s2: "skip_block_zeroing"))
3167	pf->zero_new_blocks = false;
3168
3169	else if (!strcasecmp(s1: arg_name, s2: "ignore_discard"))
3170	pf->discard_enabled = false;
3171
3172	else if (!strcasecmp(s1: arg_name, s2: "no_discard_passdown"))
3173	pf->discard_passdown = false;
3174
3175	else if (!strcasecmp(s1: arg_name, s2: "read_only"))
3176	pf->mode = PM_READ_ONLY;
3177
3178	else if (!strcasecmp(s1: arg_name, s2: "error_if_no_space"))
3179	pf->error_if_no_space = true;
3180
3181	else {
3182	ti->error = "Unrecognised pool feature requested";
3183	r = -EINVAL;
3184	break;
3185	}
3186	}
3187
3188	return r;
3189	}
3190
3191	static void metadata_low_callback(void *context)
3192	{
3193	struct pool *pool = context;
3194
3195	DMWARN("%s: reached low water mark for metadata device: sending event.",
3196	dm_device_name(pool->pool_md));
3197
3198	dm_table_event(t: pool->ti->table);
3199	}
3200
3201	/*
3202	* We need to flush the data device **before** committing the metadata.
3203	*
3204	* This ensures that the data blocks of any newly inserted mappings are
3205	* properly written to non-volatile storage and won't be lost in case of a
3206	* crash.
3207	*
3208	* Failure to do so can result in data corruption in the case of internal or
3209	* external snapshots and in the case of newly provisioned blocks, when block
3210	* zeroing is enabled.
3211	*/
3212	static int metadata_pre_commit_callback(void *context)
3213	{
3214	struct pool *pool = context;
3215
3216	return blkdev_issue_flush(bdev: pool->data_dev);
3217	}
3218
3219	static sector_t get_dev_size(struct block_device *bdev)
3220	{
3221	return bdev_nr_sectors(bdev);
3222	}
3223
3224	static void warn_if_metadata_device_too_big(struct block_device *bdev)
3225	{
3226	sector_t metadata_dev_size = get_dev_size(bdev);
3227
3228	if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
3229	DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
3230	bdev, THIN_METADATA_MAX_SECTORS);
3231	}
3232
3233	static sector_t get_metadata_dev_size(struct block_device *bdev)
3234	{
3235	sector_t metadata_dev_size = get_dev_size(bdev);
3236
3237	if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
3238	metadata_dev_size = THIN_METADATA_MAX_SECTORS;
3239
3240	return metadata_dev_size;
3241	}
3242
3243	static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
3244	{
3245	sector_t metadata_dev_size = get_metadata_dev_size(bdev);
3246
3247	sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
3248
3249	return metadata_dev_size;
3250	}
3251
3252	/*
3253	* When a metadata threshold is crossed a dm event is triggered, and
3254	* userland should respond by growing the metadata device. We could let
3255	* userland set the threshold, like we do with the data threshold, but I'm
3256	* not sure they know enough to do this well.
3257	*/
3258	static dm_block_t calc_metadata_threshold(struct pool_c *pt)
3259	{
3260	/*
3261	* 4M is ample for all ops with the possible exception of thin
3262	* device deletion which is harmless if it fails (just retry the
3263	* delete after you've grown the device).
3264	*/
3265	dm_block_t quarter = get_metadata_dev_size_in_blocks(bdev: pt->metadata_dev->bdev) / 4;
3266
3267	return min((dm_block_t)1024ULL /* 4M */, quarter);
3268	}
3269
3270	/*
3271	* thin-pool <metadata dev> <data dev>
3272	* <data block size (sectors)>
3273	* <low water mark (blocks)>
3274	* [<#feature args> [<arg>]*]
3275	*
3276	* Optional feature arguments are:
3277	* skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
3278	* ignore_discard: disable discard
3279	* no_discard_passdown: don't pass discards down to the data device
3280	* read_only: Don't allow any changes to be made to the pool metadata.
3281	* error_if_no_space: error IOs, instead of queueing, if no space.
3282	*/
3283	static int pool_ctr(struct dm_target *ti, unsigned int argc, char **argv)
3284	{
3285	int r, pool_created = 0;
3286	struct pool_c *pt;
3287	struct pool *pool;
3288	struct pool_features pf;
3289	struct dm_arg_set as;
3290	struct dm_dev *data_dev;
3291	unsigned long block_size;
3292	dm_block_t low_water_blocks;
3293	struct dm_dev *metadata_dev;
3294	blk_mode_t metadata_mode;
3295
3296	/*
3297	* FIXME Remove validation from scope of lock.
3298	*/
3299	mutex_lock(&dm_thin_pool_table.mutex);
3300
3301	if (argc < 4) {
3302	ti->error = "Invalid argument count";
3303	r = -EINVAL;
3304	goto out_unlock;
3305	}
3306
3307	as.argc = argc;
3308	as.argv = argv;
3309
3310	/* make sure metadata and data are different devices */
3311	if (!strcmp(argv[0], argv[1])) {
3312	ti->error = "Error setting metadata or data device";
3313	r = -EINVAL;
3314	goto out_unlock;
3315	}
3316
3317	/*
3318	* Set default pool features.
3319	*/
3320	pool_features_init(pf: &pf);
3321
3322	dm_consume_args(as: &as, num_args: 4);
3323	r = parse_pool_features(as: &as, pf: &pf, ti);
3324	if (r)
3325	goto out_unlock;
3326
3327	metadata_mode = BLK_OPEN_READ |
3328	((pf.mode == PM_READ_ONLY) ? 0 : BLK_OPEN_WRITE);
3329	r = dm_get_device(ti, path: argv[0], mode: metadata_mode, result: &metadata_dev);
3330	if (r) {
3331	ti->error = "Error opening metadata block device";
3332	goto out_unlock;
3333	}
3334	warn_if_metadata_device_too_big(bdev: metadata_dev->bdev);
3335
3336	r = dm_get_device(ti, path: argv[1], BLK_OPEN_READ | BLK_OPEN_WRITE, result: &data_dev);
3337	if (r) {
3338	ti->error = "Error getting data device";
3339	goto out_metadata;
3340	}
3341
3342	if (kstrtoul(s: argv[2], base: 10, res: &block_size) || !block_size ||
3343	block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
3344	block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
3345	block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
3346	ti->error = "Invalid block size";
3347	r = -EINVAL;
3348	goto out;
3349	}
3350
3351	if (kstrtoull(s: argv[3], base: 10, res: (unsigned long long *)&low_water_blocks)) {
3352	ti->error = "Invalid low water mark";
3353	r = -EINVAL;
3354	goto out;
3355	}
3356
3357	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
3358	if (!pt) {
3359	r = -ENOMEM;
3360	goto out;
3361	}
3362
3363	pool = __pool_find(pool_md: dm_table_get_md(t: ti->table), metadata_dev: metadata_dev->bdev, data_dev: data_dev->bdev,
3364	block_size, read_only: pf.mode == PM_READ_ONLY, error: &ti->error, created: &pool_created);
3365	if (IS_ERR(ptr: pool)) {
3366	r = PTR_ERR(ptr: pool);
3367	goto out_free_pt;
3368	}
3369
3370	/*
3371	* 'pool_created' reflects whether this is the first table load.
3372	* Top level discard support is not allowed to be changed after
3373	* initial load. This would require a pool reload to trigger thin
3374	* device changes.
3375	*/
3376	if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
3377	ti->error = "Discard support cannot be disabled once enabled";
3378	r = -EINVAL;
3379	goto out_flags_changed;
3380	}
3381
3382	pt->pool = pool;
3383	pt->ti = ti;
3384	pt->metadata_dev = metadata_dev;
3385	pt->data_dev = data_dev;
3386	pt->low_water_blocks = low_water_blocks;
3387	pt->adjusted_pf = pt->requested_pf = pf;
3388	ti->num_flush_bios = 1;
3389	ti->limit_swap_bios = true;
3390
3391	/*
3392	* Only need to enable discards if the pool should pass
3393	* them down to the data device. The thin device's discard
3394	* processing will cause mappings to be removed from the btree.
3395	*/
3396	if (pf.discard_enabled && pf.discard_passdown) {
3397	ti->num_discard_bios = 1;
3398	/*
3399	* Setting 'discards_supported' circumvents the normal
3400	* stacking of discard limits (this keeps the pool and
3401	* thin devices' discard limits consistent).
3402	*/
3403	ti->discards_supported = true;
3404	ti->max_discard_granularity = true;
3405	}
3406	ti->private = pt;
3407
3408	r = dm_pool_register_metadata_threshold(pmd: pt->pool->pmd,
3409	threshold: calc_metadata_threshold(pt),
3410	fn: metadata_low_callback,
3411	context: pool);
3412	if (r) {
3413	ti->error = "Error registering metadata threshold";
3414	goto out_flags_changed;
3415	}
3416
3417	dm_pool_register_pre_commit_callback(pmd: pool->pmd,
3418	fn: metadata_pre_commit_callback, context: pool);
3419
3420	mutex_unlock(lock: &dm_thin_pool_table.mutex);
3421
3422	return 0;
3423
3424	out_flags_changed:
3425	__pool_dec(pool);
3426	out_free_pt:
3427	kfree(objp: pt);
3428	out:
3429	dm_put_device(ti, d: data_dev);
3430	out_metadata:
3431	dm_put_device(ti, d: metadata_dev);
3432	out_unlock:
3433	mutex_unlock(lock: &dm_thin_pool_table.mutex);
3434
3435	return r;
3436	}
3437
3438	static int pool_map(struct dm_target *ti, struct bio *bio)
3439	{
3440	struct pool_c *pt = ti->private;
3441	struct pool *pool = pt->pool;
3442
3443	/*
3444	* As this is a singleton target, ti->begin is always zero.
3445	*/
3446	spin_lock_irq(lock: &pool->lock);
3447	bio_set_dev(bio, bdev: pt->data_dev->bdev);
3448	spin_unlock_irq(lock: &pool->lock);
3449
3450	return DM_MAPIO_REMAPPED;
3451	}
3452
3453	static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
3454	{
3455	int r;
3456	struct pool_c *pt = ti->private;
3457	struct pool *pool = pt->pool;
3458	sector_t data_size = ti->len;
3459	dm_block_t sb_data_size;
3460
3461	*need_commit = false;
3462
3463	(void) sector_div(data_size, pool->sectors_per_block);
3464
3465	r = dm_pool_get_data_dev_size(pmd: pool->pmd, result: &sb_data_size);
3466	if (r) {
3467	DMERR("%s: failed to retrieve data device size",
3468	dm_device_name(pool->pool_md));
3469	return r;
3470	}
3471
3472	if (data_size < sb_data_size) {
3473	DMERR("%s: pool target (%llu blocks) too small: expected %llu",
3474	dm_device_name(pool->pool_md),
3475	(unsigned long long)data_size, sb_data_size);
3476	return -EINVAL;
3477
3478	} else if (data_size > sb_data_size) {
3479	if (dm_pool_metadata_needs_check(pmd: pool->pmd)) {
3480	DMERR("%s: unable to grow the data device until repaired.",
3481	dm_device_name(pool->pool_md));
3482	return 0;
3483	}
3484
3485	if (sb_data_size)
3486	DMINFO("%s: growing the data device from %llu to %llu blocks",
3487	dm_device_name(pool->pool_md),
3488	sb_data_size, (unsigned long long)data_size);
3489	r = dm_pool_resize_data_dev(pmd: pool->pmd, new_size: data_size);
3490	if (r) {
3491	metadata_operation_failed(pool, op: "dm_pool_resize_data_dev", r);
3492	return r;
3493	}
3494
3495	*need_commit = true;
3496	}
3497
3498	return 0;
3499	}
3500
3501	static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
3502	{
3503	int r;
3504	struct pool_c *pt = ti->private;
3505	struct pool *pool = pt->pool;
3506	dm_block_t metadata_dev_size, sb_metadata_dev_size;
3507
3508	*need_commit = false;
3509
3510	metadata_dev_size = get_metadata_dev_size_in_blocks(bdev: pool->md_dev);
3511
3512	r = dm_pool_get_metadata_dev_size(pmd: pool->pmd, result: &sb_metadata_dev_size);
3513	if (r) {
3514	DMERR("%s: failed to retrieve metadata device size",
3515	dm_device_name(pool->pool_md));
3516	return r;
3517	}
3518
3519	if (metadata_dev_size < sb_metadata_dev_size) {
3520	DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
3521	dm_device_name(pool->pool_md),
3522	metadata_dev_size, sb_metadata_dev_size);
3523	return -EINVAL;
3524
3525	} else if (metadata_dev_size > sb_metadata_dev_size) {
3526	if (dm_pool_metadata_needs_check(pmd: pool->pmd)) {
3527	DMERR("%s: unable to grow the metadata device until repaired.",
3528	dm_device_name(pool->pool_md));
3529	return 0;
3530	}
3531
3532	warn_if_metadata_device_too_big(bdev: pool->md_dev);
3533	DMINFO("%s: growing the metadata device from %llu to %llu blocks",
3534	dm_device_name(pool->pool_md),
3535	sb_metadata_dev_size, metadata_dev_size);
3536
3537	if (get_pool_mode(pool) == PM_OUT_OF_METADATA_SPACE)
3538	set_pool_mode(pool, new_mode: PM_WRITE);
3539
3540	r = dm_pool_resize_metadata_dev(pmd: pool->pmd, new_size: metadata_dev_size);
3541	if (r) {
3542	metadata_operation_failed(pool, op: "dm_pool_resize_metadata_dev", r);
3543	return r;
3544	}
3545
3546	*need_commit = true;
3547	}
3548
3549	return 0;
3550	}
3551
3552	/*
3553	* Retrieves the number of blocks of the data device from
3554	* the superblock and compares it to the actual device size,
3555	* thus resizing the data device in case it has grown.
3556	*
3557	* This both copes with opening preallocated data devices in the ctr
3558	* being followed by a resume
3559	* -and-
3560	* calling the resume method individually after userspace has
3561	* grown the data device in reaction to a table event.
3562	*/
3563	static int pool_preresume(struct dm_target *ti)
3564	{
3565	int r;
3566	bool need_commit1, need_commit2;
3567	struct pool_c *pt = ti->private;
3568	struct pool *pool = pt->pool;
3569
3570	/*
3571	* Take control of the pool object.
3572	*/
3573	r = bind_control_target(pool, ti);
3574	if (r)
3575	goto out;
3576
3577	r = maybe_resize_data_dev(ti, need_commit: &need_commit1);
3578	if (r)
3579	goto out;
3580
3581	r = maybe_resize_metadata_dev(ti, need_commit: &need_commit2);
3582	if (r)
3583	goto out;
3584
3585	if (need_commit1 || need_commit2)
3586	(void) commit(pool);
3587	out:
3588	/*
3589	* When a thin-pool is PM_FAIL, it cannot be rebuilt if
3590	* bio is in deferred list. Therefore need to return 0
3591	* to allow pool_resume() to flush IO.
3592	*/
3593	if (r && get_pool_mode(pool) == PM_FAIL)
3594	r = 0;
3595
3596	return r;
3597	}
3598
3599	static void pool_suspend_active_thins(struct pool *pool)
3600	{
3601	struct thin_c *tc;
3602
3603	/* Suspend all active thin devices */
3604	tc = get_first_thin(pool);
3605	while (tc) {
3606	dm_internal_suspend_noflush(md: tc->thin_md);
3607	tc = get_next_thin(pool, tc);
3608	}
3609	}
3610
3611	static void pool_resume_active_thins(struct pool *pool)
3612	{
3613	struct thin_c *tc;
3614
3615	/* Resume all active thin devices */
3616	tc = get_first_thin(pool);
3617	while (tc) {
3618	dm_internal_resume(md: tc->thin_md);
3619	tc = get_next_thin(pool, tc);
3620	}
3621	}
3622
3623	static void pool_resume(struct dm_target *ti)
3624	{
3625	struct pool_c *pt = ti->private;
3626	struct pool *pool = pt->pool;
3627
3628	/*
3629	* Must requeue active_thins' bios and then resume
3630	* active_thins _before_ clearing 'suspend' flag.
3631	*/
3632	requeue_bios(pool);
3633	pool_resume_active_thins(pool);
3634
3635	spin_lock_irq(lock: &pool->lock);
3636	pool->low_water_triggered = false;
3637	pool->suspended = false;
3638	spin_unlock_irq(lock: &pool->lock);
3639
3640	do_waker(ws: &pool->waker.work);
3641	}
3642
3643	static void pool_presuspend(struct dm_target *ti)
3644	{
3645	struct pool_c *pt = ti->private;
3646	struct pool *pool = pt->pool;
3647
3648	spin_lock_irq(lock: &pool->lock);
3649	pool->suspended = true;
3650	spin_unlock_irq(lock: &pool->lock);
3651
3652	pool_suspend_active_thins(pool);
3653	}
3654
3655	static void pool_presuspend_undo(struct dm_target *ti)
3656	{
3657	struct pool_c *pt = ti->private;
3658	struct pool *pool = pt->pool;
3659
3660	pool_resume_active_thins(pool);
3661
3662	spin_lock_irq(lock: &pool->lock);
3663	pool->suspended = false;
3664	spin_unlock_irq(lock: &pool->lock);
3665	}
3666
3667	static void pool_postsuspend(struct dm_target *ti)
3668	{
3669	struct pool_c *pt = ti->private;
3670	struct pool *pool = pt->pool;
3671
3672	cancel_delayed_work_sync(dwork: &pool->waker);
3673	cancel_delayed_work_sync(dwork: &pool->no_space_timeout);
3674	flush_workqueue(pool->wq);
3675	(void) commit(pool);
3676	}
3677
3678	static int check_arg_count(unsigned int argc, unsigned int args_required)
3679	{
3680	if (argc != args_required) {
3681	DMWARN("Message received with %u arguments instead of %u.",
3682	argc, args_required);
3683	return -EINVAL;
3684	}
3685
3686	return 0;
3687	}
3688
3689	static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
3690	{
3691	if (!kstrtoull(s: arg, base: 10, res: (unsigned long long *)dev_id) &&
3692	*dev_id <= MAX_DEV_ID)
3693	return 0;
3694
3695	if (warning)
3696	DMWARN("Message received with invalid device id: %s", arg);
3697
3698	return -EINVAL;
3699	}
3700
3701	static int process_create_thin_mesg(unsigned int argc, char **argv, struct pool *pool)
3702	{
3703	dm_thin_id dev_id;
3704	int r;
3705
3706	r = check_arg_count(argc, args_required: 2);
3707	if (r)
3708	return r;
3709
3710	r = read_dev_id(arg: argv[1], dev_id: &dev_id, warning: 1);
3711	if (r)
3712	return r;
3713
3714	r = dm_pool_create_thin(pmd: pool->pmd, dev: dev_id);
3715	if (r) {
3716	DMWARN("Creation of new thinly-provisioned device with id %s failed.",
3717	argv[1]);
3718	return r;
3719	}
3720
3721	return 0;
3722	}
3723
3724	static int process_create_snap_mesg(unsigned int argc, char **argv, struct pool *pool)
3725	{
3726	dm_thin_id dev_id;
3727	dm_thin_id origin_dev_id;
3728	int r;
3729
3730	r = check_arg_count(argc, args_required: 3);
3731	if (r)
3732	return r;
3733
3734	r = read_dev_id(arg: argv[1], dev_id: &dev_id, warning: 1);
3735	if (r)
3736	return r;
3737
3738	r = read_dev_id(arg: argv[2], dev_id: &origin_dev_id, warning: 1);
3739	if (r)
3740	return r;
3741
3742	r = dm_pool_create_snap(pmd: pool->pmd, dev: dev_id, origin: origin_dev_id);
3743	if (r) {
3744	DMWARN("Creation of new snapshot %s of device %s failed.",
3745	argv[1], argv[2]);
3746	return r;
3747	}
3748
3749	return 0;
3750	}
3751
3752	static int process_delete_mesg(unsigned int argc, char **argv, struct pool *pool)
3753	{
3754	dm_thin_id dev_id;
3755	int r;
3756
3757	r = check_arg_count(argc, args_required: 2);
3758	if (r)
3759	return r;
3760
3761	r = read_dev_id(arg: argv[1], dev_id: &dev_id, warning: 1);
3762	if (r)
3763	return r;
3764
3765	r = dm_pool_delete_thin_device(pmd: pool->pmd, dev: dev_id);
3766	if (r)
3767	DMWARN("Deletion of thin device %s failed.", argv[1]);
3768
3769	return r;
3770	}
3771
3772	static int process_set_transaction_id_mesg(unsigned int argc, char **argv, struct pool *pool)
3773	{
3774	dm_thin_id old_id, new_id;
3775	int r;
3776
3777	r = check_arg_count(argc, args_required: 3);
3778	if (r)
3779	return r;
3780
3781	if (kstrtoull(s: argv[1], base: 10, res: (unsigned long long *)&old_id)) {
3782	DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
3783	return -EINVAL;
3784	}
3785
3786	if (kstrtoull(s: argv[2], base: 10, res: (unsigned long long *)&new_id)) {
3787	DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
3788	return -EINVAL;
3789	}
3790
3791	r = dm_pool_set_metadata_transaction_id(pmd: pool->pmd, current_id: old_id, new_id);
3792	if (r) {
3793	DMWARN("Failed to change transaction id from %s to %s.",
3794	argv[1], argv[2]);
3795	return r;
3796	}
3797
3798	return 0;
3799	}
3800
3801	static int process_reserve_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool)
3802	{
3803	int r;
3804
3805	r = check_arg_count(argc, args_required: 1);
3806	if (r)
3807	return r;
3808
3809	(void) commit(pool);
3810
3811	r = dm_pool_reserve_metadata_snap(pmd: pool->pmd);
3812	if (r)
3813	DMWARN("reserve_metadata_snap message failed.");
3814
3815	return r;
3816	}
3817
3818	static int process_release_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool)
3819	{
3820	int r;
3821
3822	r = check_arg_count(argc, args_required: 1);
3823	if (r)
3824	return r;
3825
3826	r = dm_pool_release_metadata_snap(pmd: pool->pmd);
3827	if (r)
3828	DMWARN("release_metadata_snap message failed.");
3829
3830	return r;
3831	}
3832
3833	/*
3834	* Messages supported:
3835	* create_thin <dev_id>
3836	* create_snap <dev_id> <origin_id>
3837	* delete <dev_id>
3838	* set_transaction_id <current_trans_id> <new_trans_id>
3839	* reserve_metadata_snap
3840	* release_metadata_snap
3841	*/
3842	static int pool_message(struct dm_target *ti, unsigned int argc, char **argv,
3843	char *result, unsigned int maxlen)
3844	{
3845	int r = -EINVAL;
3846	struct pool_c *pt = ti->private;
3847	struct pool *pool = pt->pool;
3848
3849	if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) {
3850	DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
3851	dm_device_name(pool->pool_md));
3852	return -EOPNOTSUPP;
3853	}
3854
3855	if (!strcasecmp(s1: argv[0], s2: "create_thin"))
3856	r = process_create_thin_mesg(argc, argv, pool);
3857
3858	else if (!strcasecmp(s1: argv[0], s2: "create_snap"))
3859	r = process_create_snap_mesg(argc, argv, pool);
3860
3861	else if (!strcasecmp(s1: argv[0], s2: "delete"))
3862	r = process_delete_mesg(argc, argv, pool);
3863
3864	else if (!strcasecmp(s1: argv[0], s2: "set_transaction_id"))
3865	r = process_set_transaction_id_mesg(argc, argv, pool);
3866
3867	else if (!strcasecmp(s1: argv[0], s2: "reserve_metadata_snap"))
3868	r = process_reserve_metadata_snap_mesg(argc, argv, pool);
3869
3870	else if (!strcasecmp(s1: argv[0], s2: "release_metadata_snap"))
3871	r = process_release_metadata_snap_mesg(argc, argv, pool);
3872
3873	else
3874	DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
3875
3876	if (!r)
3877	(void) commit(pool);
3878
3879	return r;
3880	}
3881
3882	static void emit_flags(struct pool_features *pf, char *result,
3883	unsigned int sz, unsigned int maxlen)
3884	{
3885	unsigned int count = !pf->zero_new_blocks + !pf->discard_enabled +
3886	!pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
3887	pf->error_if_no_space;
3888	DMEMIT("%u ", count);
3889
3890	if (!pf->zero_new_blocks)
3891	DMEMIT("skip_block_zeroing ");
3892
3893	if (!pf->discard_enabled)
3894	DMEMIT("ignore_discard ");
3895
3896	if (!pf->discard_passdown)
3897	DMEMIT("no_discard_passdown ");
3898
3899	if (pf->mode == PM_READ_ONLY)
3900	DMEMIT("read_only ");
3901
3902	if (pf->error_if_no_space)
3903	DMEMIT("error_if_no_space ");
3904	}
3905
3906	/*
3907	* Status line is:
3908	* <transaction id> <used metadata sectors>/<total metadata sectors>
3909	* <used data sectors>/<total data sectors> <held metadata root>
3910	* <pool mode> <discard config> <no space config> <needs_check>
3911	*/
3912	static void pool_status(struct dm_target *ti, status_type_t type,
3913	unsigned int status_flags, char *result, unsigned int maxlen)
3914	{
3915	int r;
3916	unsigned int sz = 0;
3917	uint64_t transaction_id;
3918	dm_block_t nr_free_blocks_data;
3919	dm_block_t nr_free_blocks_metadata;
3920	dm_block_t nr_blocks_data;
3921	dm_block_t nr_blocks_metadata;
3922	dm_block_t held_root;
3923	enum pool_mode mode;
3924	char buf[BDEVNAME_SIZE];
3925	char buf2[BDEVNAME_SIZE];
3926	struct pool_c *pt = ti->private;
3927	struct pool *pool = pt->pool;
3928
3929	switch (type) {
3930	case STATUSTYPE_INFO:
3931	if (get_pool_mode(pool) == PM_FAIL) {
3932	DMEMIT("Fail");
3933	break;
3934	}
3935
3936	/* Commit to ensure statistics aren't out-of-date */
3937	if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3938	(void) commit(pool);
3939
3940	r = dm_pool_get_metadata_transaction_id(pmd: pool->pmd, result: &transaction_id);
3941	if (r) {
3942	DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3943	dm_device_name(pool->pool_md), r);
3944	goto err;
3945	}
3946
3947	r = dm_pool_get_free_metadata_block_count(pmd: pool->pmd, result: &nr_free_blocks_metadata);
3948	if (r) {
3949	DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3950	dm_device_name(pool->pool_md), r);
3951	goto err;
3952	}
3953
3954	r = dm_pool_get_metadata_dev_size(pmd: pool->pmd, result: &nr_blocks_metadata);
3955	if (r) {
3956	DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3957	dm_device_name(pool->pool_md), r);
3958	goto err;
3959	}
3960
3961	r = dm_pool_get_free_block_count(pmd: pool->pmd, result: &nr_free_blocks_data);
3962	if (r) {
3963	DMERR("%s: dm_pool_get_free_block_count returned %d",
3964	dm_device_name(pool->pool_md), r);
3965	goto err;
3966	}
3967
3968	r = dm_pool_get_data_dev_size(pmd: pool->pmd, result: &nr_blocks_data);
3969	if (r) {
3970	DMERR("%s: dm_pool_get_data_dev_size returned %d",
3971	dm_device_name(pool->pool_md), r);
3972	goto err;
3973	}
3974
3975	r = dm_pool_get_metadata_snap(pmd: pool->pmd, result: &held_root);
3976	if (r) {
3977	DMERR("%s: dm_pool_get_metadata_snap returned %d",
3978	dm_device_name(pool->pool_md), r);
3979	goto err;
3980	}
3981
3982	DMEMIT("%llu %llu/%llu %llu/%llu ",
3983	(unsigned long long)transaction_id,
3984	(unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3985	(unsigned long long)nr_blocks_metadata,
3986	(unsigned long long)(nr_blocks_data - nr_free_blocks_data),
3987	(unsigned long long)nr_blocks_data);
3988
3989	if (held_root)
3990	DMEMIT("%llu ", held_root);
3991	else
3992	DMEMIT("- ");
3993
3994	mode = get_pool_mode(pool);
3995	if (mode == PM_OUT_OF_DATA_SPACE)
3996	DMEMIT("out_of_data_space ");
3997	else if (is_read_only_pool_mode(mode))
3998	DMEMIT("ro ");
3999	else
4000	DMEMIT("rw ");
4001
4002	if (!pool->pf.discard_enabled)
4003	DMEMIT("ignore_discard ");
4004	else if (pool->pf.discard_passdown)
4005	DMEMIT("discard_passdown ");
4006	else
4007	DMEMIT("no_discard_passdown ");
4008
4009	if (pool->pf.error_if_no_space)
4010	DMEMIT("error_if_no_space ");
4011	else
4012	DMEMIT("queue_if_no_space ");
4013
4014	if (dm_pool_metadata_needs_check(pmd: pool->pmd))
4015	DMEMIT("needs_check ");
4016	else
4017	DMEMIT("- ");
4018
4019	DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt));
4020
4021	break;
4022
4023	case STATUSTYPE_TABLE:
4024	DMEMIT("%s %s %lu %llu ",
4025	format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
4026	format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
4027	(unsigned long)pool->sectors_per_block,
4028	(unsigned long long)pt->low_water_blocks);
4029	emit_flags(pf: &pt->requested_pf, result, sz, maxlen);
4030	break;
4031
4032	case STATUSTYPE_IMA:
4033	*result = '\0';
4034	break;
4035	}
4036	return;
4037
4038	err:
4039	DMEMIT("Error");
4040	}
4041
4042	static int pool_iterate_devices(struct dm_target *ti,
4043	iterate_devices_callout_fn fn, void *data)
4044	{
4045	struct pool_c *pt = ti->private;
4046
4047	return fn(ti, pt->data_dev, 0, ti->len, data);
4048	}
4049
4050	static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
4051	{
4052	struct pool_c *pt = ti->private;
4053	struct pool *pool = pt->pool;
4054	sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
4055
4056	/*
4057	* If max_sectors is smaller than pool->sectors_per_block adjust it
4058	* to the highest possible power-of-2 factor of pool->sectors_per_block.
4059	* This is especially beneficial when the pool's data device is a RAID
4060	* device that has a full stripe width that matches pool->sectors_per_block
4061	* -- because even though partial RAID stripe-sized IOs will be issued to a
4062	* single RAID stripe; when aggregated they will end on a full RAID stripe
4063	* boundary.. which avoids additional partial RAID stripe writes cascading
4064	*/
4065	if (limits->max_sectors < pool->sectors_per_block) {
4066	while (!is_factor(block_size: pool->sectors_per_block, n: limits->max_sectors)) {
4067	if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
4068	limits->max_sectors--;
4069	limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
4070	}
4071	}
4072
4073	/*
4074	* If the system-determined stacked limits are compatible with the
4075	* pool's blocksize (io_opt is a factor) do not override them.
4076	*/
4077	if (io_opt_sectors < pool->sectors_per_block ||
4078	!is_factor(block_size: io_opt_sectors, n: pool->sectors_per_block)) {
4079	if (is_factor(block_size: pool->sectors_per_block, n: limits->max_sectors))
4080	limits->io_min = limits->max_sectors << SECTOR_SHIFT;
4081	else
4082	limits->io_min = pool->sectors_per_block << SECTOR_SHIFT;
4083	limits->io_opt = pool->sectors_per_block << SECTOR_SHIFT;
4084	}
4085
4086	/*
4087	* pt->adjusted_pf is a staging area for the actual features to use.
4088	* They get transferred to the live pool in bind_control_target()
4089	* called from pool_preresume().
4090	*/
4091
4092	if (pt->adjusted_pf.discard_enabled) {
4093	disable_discard_passdown_if_not_supported(pt);
4094	if (!pt->adjusted_pf.discard_passdown)
4095	limits->max_hw_discard_sectors = 0;
4096	/*
4097	* The pool uses the same discard limits as the underlying data
4098	* device. DM core has already set this up.
4099	*/
4100	} else {
4101	/*
4102	* Must explicitly disallow stacking discard limits otherwise the
4103	* block layer will stack them if pool's data device has support.
4104	*/
4105	limits->discard_granularity = 0;
4106	}
4107	}
4108
4109	static struct target_type pool_target = {
4110	.name = "thin-pool",
4111	.features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
4112	DM_TARGET_IMMUTABLE | DM_TARGET_PASSES_CRYPTO,
4113	.version = {1, 24, 0},
4114	.module = THIS_MODULE,
4115	.ctr = pool_ctr,
4116	.dtr = pool_dtr,
4117	.map = pool_map,
4118	.presuspend = pool_presuspend,
4119	.presuspend_undo = pool_presuspend_undo,
4120	.postsuspend = pool_postsuspend,
4121	.preresume = pool_preresume,
4122	.resume = pool_resume,
4123	.message = pool_message,
4124	.status = pool_status,
4125	.iterate_devices = pool_iterate_devices,
4126	.io_hints = pool_io_hints,
4127	};
4128
4129	/*
4130	*--------------------------------------------------------------
4131	* Thin target methods
4132	*--------------------------------------------------------------
4133	*/
4134	static void thin_get(struct thin_c *tc)
4135	{
4136	refcount_inc(r: &tc->refcount);
4137	}
4138
4139	static void thin_put(struct thin_c *tc)
4140	{
4141	if (refcount_dec_and_test(r: &tc->refcount))
4142	complete(&tc->can_destroy);
4143	}
4144
4145	static void thin_dtr(struct dm_target *ti)
4146	{
4147	struct thin_c *tc = ti->private;
4148
4149	spin_lock_irq(lock: &tc->pool->lock);
4150	list_del_rcu(entry: &tc->list);
4151	spin_unlock_irq(lock: &tc->pool->lock);
4152	synchronize_rcu();
4153
4154	thin_put(tc);
4155	wait_for_completion(&tc->can_destroy);
4156
4157	mutex_lock(&dm_thin_pool_table.mutex);
4158
4159	__pool_dec(pool: tc->pool);
4160	dm_pool_close_thin_device(td: tc->td);
4161	dm_put_device(ti, d: tc->pool_dev);
4162	if (tc->origin_dev)
4163	dm_put_device(ti, d: tc->origin_dev);
4164	kfree(objp: tc);
4165
4166	mutex_unlock(lock: &dm_thin_pool_table.mutex);
4167	}
4168
4169	/*
4170	* Thin target parameters:
4171	*
4172	* <pool_dev> <dev_id> [origin_dev]
4173	*
4174	* pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
4175	* dev_id: the internal device identifier
4176	* origin_dev: a device external to the pool that should act as the origin
4177	*
4178	* If the pool device has discards disabled, they get disabled for the thin
4179	* device as well.
4180	*/
4181	static int thin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
4182	{
4183	int r;
4184	struct thin_c *tc;
4185	struct dm_dev *pool_dev, *origin_dev;
4186	struct mapped_device *pool_md;
4187
4188	mutex_lock(&dm_thin_pool_table.mutex);
4189
4190	if (argc != 2 && argc != 3) {
4191	ti->error = "Invalid argument count";
4192	r = -EINVAL;
4193	goto out_unlock;
4194	}
4195
4196	tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
4197	if (!tc) {
4198	ti->error = "Out of memory";
4199	r = -ENOMEM;
4200	goto out_unlock;
4201	}
4202	tc->thin_md = dm_table_get_md(t: ti->table);
4203	spin_lock_init(&tc->lock);
4204	INIT_LIST_HEAD(list: &tc->deferred_cells);
4205	bio_list_init(bl: &tc->deferred_bio_list);
4206	bio_list_init(bl: &tc->retry_on_resume_list);
4207	tc->sort_bio_list = RB_ROOT;
4208
4209	if (argc == 3) {
4210	if (!strcmp(argv[0], argv[2])) {
4211	ti->error = "Error setting origin device";
4212	r = -EINVAL;
4213	goto bad_origin_dev;
4214	}
4215
4216	r = dm_get_device(ti, path: argv[2], BLK_OPEN_READ, result: &origin_dev);
4217	if (r) {
4218	ti->error = "Error opening origin device";
4219	goto bad_origin_dev;
4220	}
4221	tc->origin_dev = origin_dev;
4222	}
4223
4224	r = dm_get_device(ti, path: argv[0], mode: dm_table_get_mode(t: ti->table), result: &pool_dev);
4225	if (r) {
4226	ti->error = "Error opening pool device";
4227	goto bad_pool_dev;
4228	}
4229	tc->pool_dev = pool_dev;
4230
4231	if (read_dev_id(arg: argv[1], dev_id: (unsigned long long *)&tc->dev_id, warning: 0)) {
4232	ti->error = "Invalid device id";
4233	r = -EINVAL;
4234	goto bad_common;
4235	}
4236
4237	pool_md = dm_get_md(dev: tc->pool_dev->bdev->bd_dev);
4238	if (!pool_md) {
4239	ti->error = "Couldn't get pool mapped device";
4240	r = -EINVAL;
4241	goto bad_common;
4242	}
4243
4244	tc->pool = __pool_table_lookup(md: pool_md);
4245	if (!tc->pool) {
4246	ti->error = "Couldn't find pool object";
4247	r = -EINVAL;
4248	goto bad_pool_lookup;
4249	}
4250	__pool_inc(pool: tc->pool);
4251
4252	if (get_pool_mode(pool: tc->pool) == PM_FAIL) {
4253	ti->error = "Couldn't open thin device, Pool is in fail mode";
4254	r = -EINVAL;
4255	goto bad_pool;
4256	}
4257
4258	r = dm_pool_open_thin_device(pmd: tc->pool->pmd, dev: tc->dev_id, td: &tc->td);
4259	if (r) {
4260	ti->error = "Couldn't open thin internal device";
4261	goto bad_pool;
4262	}
4263
4264	r = dm_set_target_max_io_len(ti, len: tc->pool->sectors_per_block);
4265	if (r)
4266	goto bad;
4267
4268	ti->num_flush_bios = 1;
4269	ti->limit_swap_bios = true;
4270	ti->flush_supported = true;
4271	ti->accounts_remapped_io = true;
4272	ti->per_io_data_size = sizeof(struct dm_thin_endio_hook);
4273
4274	/* In case the pool supports discards, pass them on. */
4275	if (tc->pool->pf.discard_enabled) {
4276	ti->discards_supported = true;
4277	ti->num_discard_bios = 1;
4278	ti->max_discard_granularity = true;
4279	}
4280
4281	mutex_unlock(lock: &dm_thin_pool_table.mutex);
4282
4283	spin_lock_irq(lock: &tc->pool->lock);
4284	if (tc->pool->suspended) {
4285	spin_unlock_irq(lock: &tc->pool->lock);
4286	mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
4287	ti->error = "Unable to activate thin device while pool is suspended";
4288	r = -EINVAL;
4289	goto bad;
4290	}
4291	refcount_set(r: &tc->refcount, n: 1);
4292	init_completion(x: &tc->can_destroy);
4293	list_add_tail_rcu(new: &tc->list, head: &tc->pool->active_thins);
4294	spin_unlock_irq(lock: &tc->pool->lock);
4295	/*
4296	* This synchronize_rcu() call is needed here otherwise we risk a
4297	* wake_worker() call finding no bios to process (because the newly
4298	* added tc isn't yet visible). So this reduces latency since we
4299	* aren't then dependent on the periodic commit to wake_worker().
4300	*/
4301	synchronize_rcu();
4302
4303	dm_put(md: pool_md);
4304
4305	return 0;
4306
4307	bad:
4308	dm_pool_close_thin_device(td: tc->td);
4309	bad_pool:
4310	__pool_dec(pool: tc->pool);
4311	bad_pool_lookup:
4312	dm_put(md: pool_md);
4313	bad_common:
4314	dm_put_device(ti, d: tc->pool_dev);
4315	bad_pool_dev:
4316	if (tc->origin_dev)
4317	dm_put_device(ti, d: tc->origin_dev);
4318	bad_origin_dev:
4319	kfree(objp: tc);
4320	out_unlock:
4321	mutex_unlock(lock: &dm_thin_pool_table.mutex);
4322
4323	return r;
4324	}
4325
4326	static int thin_map(struct dm_target *ti, struct bio *bio)
4327	{
4328	bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
4329
4330	return thin_bio_map(ti, bio);
4331	}
4332
4333	static int thin_endio(struct dm_target *ti, struct bio *bio,
4334	blk_status_t *err)
4335	{
4336	unsigned long flags;
4337	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, data_size: sizeof(struct dm_thin_endio_hook));
4338	struct list_head work;
4339	struct dm_thin_new_mapping *m, *tmp;
4340	struct pool *pool = h->tc->pool;
4341
4342	if (h->shared_read_entry) {
4343	INIT_LIST_HEAD(list: &work);
4344	dm_deferred_entry_dec(entry: h->shared_read_entry, head: &work);
4345
4346	spin_lock_irqsave(&pool->lock, flags);
4347	list_for_each_entry_safe(m, tmp, &work, list) {
4348	list_del(entry: &m->list);
4349	__complete_mapping_preparation(m);
4350	}
4351	spin_unlock_irqrestore(lock: &pool->lock, flags);
4352	}
4353
4354	if (h->all_io_entry) {
4355	INIT_LIST_HEAD(list: &work);
4356	dm_deferred_entry_dec(entry: h->all_io_entry, head: &work);
4357	if (!list_empty(head: &work)) {
4358	spin_lock_irqsave(&pool->lock, flags);
4359	list_for_each_entry_safe(m, tmp, &work, list)
4360	list_add_tail(new: &m->list, head: &pool->prepared_discards);
4361	spin_unlock_irqrestore(lock: &pool->lock, flags);
4362	wake_worker(pool);
4363	}
4364	}
4365
4366	if (h->cell)
4367	cell_defer_no_holder(tc: h->tc, cell: h->cell);
4368
4369	return DM_ENDIO_DONE;
4370	}
4371
4372	static void thin_presuspend(struct dm_target *ti)
4373	{
4374	struct thin_c *tc = ti->private;
4375
4376	if (dm_noflush_suspending(ti))
4377	noflush_work(tc, fn: do_noflush_start);
4378	}
4379
4380	static void thin_postsuspend(struct dm_target *ti)
4381	{
4382	struct thin_c *tc = ti->private;
4383
4384	if (dm_noflush_suspending(ti))
4385	noflush_work(tc, fn: do_noflush_stop);
4386	}
4387
4388	static int thin_preresume(struct dm_target *ti)
4389	{
4390	struct thin_c *tc = ti->private;
4391
4392	if (tc->origin_dev)
4393	tc->origin_size = get_dev_size(bdev: tc->origin_dev->bdev);
4394
4395	return 0;
4396	}
4397
4398	/*
4399	* <nr mapped sectors> <highest mapped sector>
4400	*/
4401	static void thin_status(struct dm_target *ti, status_type_t type,
4402	unsigned int status_flags, char *result, unsigned int maxlen)
4403	{
4404	int r;
4405	ssize_t sz = 0;
4406	dm_block_t mapped, highest;
4407	char buf[BDEVNAME_SIZE];
4408	struct thin_c *tc = ti->private;
4409
4410	if (get_pool_mode(pool: tc->pool) == PM_FAIL) {
4411	DMEMIT("Fail");
4412	return;
4413	}
4414
4415	if (!tc->td)
4416	DMEMIT("-");
4417	else {
4418	switch (type) {
4419	case STATUSTYPE_INFO:
4420	r = dm_thin_get_mapped_count(td: tc->td, result: &mapped);
4421	if (r) {
4422	DMERR("dm_thin_get_mapped_count returned %d", r);
4423	goto err;
4424	}
4425
4426	r = dm_thin_get_highest_mapped_block(td: tc->td, highest_mapped: &highest);
4427	if (r < 0) {
4428	DMERR("dm_thin_get_highest_mapped_block returned %d", r);
4429	goto err;
4430	}
4431
4432	DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
4433	if (r)
4434	DMEMIT("%llu", ((highest + 1) *
4435	tc->pool->sectors_per_block) - 1);
4436	else
4437	DMEMIT("-");
4438	break;
4439
4440	case STATUSTYPE_TABLE:
4441	DMEMIT("%s %lu",
4442	format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
4443	(unsigned long) tc->dev_id);
4444	if (tc->origin_dev)
4445	DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
4446	break;
4447
4448	case STATUSTYPE_IMA:
4449	*result = '\0';
4450	break;
4451	}
4452	}
4453
4454	return;
4455
4456	err:
4457	DMEMIT("Error");
4458	}
4459
4460	static int thin_iterate_devices(struct dm_target *ti,
4461	iterate_devices_callout_fn fn, void *data)
4462	{
4463	sector_t blocks;
4464	struct thin_c *tc = ti->private;
4465	struct pool *pool = tc->pool;
4466
4467	/*
4468	* We can't call dm_pool_get_data_dev_size() since that blocks. So
4469	* we follow a more convoluted path through to the pool's target.
4470	*/
4471	if (!pool->ti)
4472	return 0; /* nothing is bound */
4473
4474	blocks = pool->ti->len;
4475	(void) sector_div(blocks, pool->sectors_per_block);
4476	if (blocks)
4477	return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
4478
4479	return 0;
4480	}
4481
4482	static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
4483	{
4484	struct thin_c *tc = ti->private;
4485	struct pool *pool = tc->pool;
4486
4487	if (pool->pf.discard_enabled) {
4488	limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
4489	limits->max_hw_discard_sectors = pool->sectors_per_block * BIO_PRISON_MAX_RANGE;
4490	}
4491	}
4492
4493	static struct target_type thin_target = {
4494	.name = "thin",
4495	.features = DM_TARGET_PASSES_CRYPTO,
4496	.version = {1, 24, 0},
4497	.module = THIS_MODULE,
4498	.ctr = thin_ctr,
4499	.dtr = thin_dtr,
4500	.map = thin_map,
4501	.end_io = thin_endio,
4502	.preresume = thin_preresume,
4503	.presuspend = thin_presuspend,
4504	.postsuspend = thin_postsuspend,
4505	.status = thin_status,
4506	.iterate_devices = thin_iterate_devices,
4507	.io_hints = thin_io_hints,
4508	};
4509
4510	/*----------------------------------------------------------------*/
4511
4512	static int __init dm_thin_init(void)
4513	{
4514	int r = -ENOMEM;
4515
4516	pool_table_init();
4517
4518	_new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
4519	if (!_new_mapping_cache)
4520	return r;
4521
4522	r = dm_register_target(t: &thin_target);
4523	if (r)
4524	goto bad_new_mapping_cache;
4525
4526	r = dm_register_target(t: &pool_target);
4527	if (r)
4528	goto bad_thin_target;
4529
4530	return 0;
4531
4532	bad_thin_target:
4533	dm_unregister_target(t: &thin_target);
4534	bad_new_mapping_cache:
4535	kmem_cache_destroy(s: _new_mapping_cache);
4536
4537	return r;
4538	}
4539
4540	static void dm_thin_exit(void)
4541	{
4542	dm_unregister_target(t: &thin_target);
4543	dm_unregister_target(t: &pool_target);
4544
4545	kmem_cache_destroy(s: _new_mapping_cache);
4546
4547	pool_table_exit();
4548	}
4549
4550	module_init(dm_thin_init);
4551	module_exit(dm_thin_exit);
4552
4553	module_param_named(no_space_timeout, no_space_timeout_secs, uint, 0644);
4554	MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
4555
4556	MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
4557	MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>");
4558	MODULE_LICENSE("GPL");
4559