1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* raid10.c : Multiple Devices driver for Linux
4	*
5	* Copyright (C) 2000-2004 Neil Brown
6	*
7	* RAID-10 support for md.
8	*
9	* Base on code in raid1.c. See raid1.c for further copyright information.
10	*/
11
12	#include <linux/slab.h>
13	#include <linux/delay.h>
14	#include <linux/blkdev.h>
15	#include <linux/module.h>
16	#include <linux/seq_file.h>
17	#include <linux/ratelimit.h>
18	#include <linux/kthread.h>
19	#include <linux/raid/md_p.h>
20	#include <trace/events/block.h>
21	#include "md.h"
22
23	#define RAID_1_10_NAME "raid10"
24	#include "raid10.h"
25	#include "raid0.h"
26	#include "md-bitmap.h"
27	#include "md-cluster.h"
28
29	/*
30	* RAID10 provides a combination of RAID0 and RAID1 functionality.
31	* The layout of data is defined by
32	* chunk_size
33	* raid_disks
34	* near_copies (stored in low byte of layout)
35	* far_copies (stored in second byte of layout)
36	* far_offset (stored in bit 16 of layout )
37	* use_far_sets (stored in bit 17 of layout )
38	* use_far_sets_bugfixed (stored in bit 18 of layout )
39	*
40	* The data to be stored is divided into chunks using chunksize. Each device
41	* is divided into far_copies sections. In each section, chunks are laid out
42	* in a style similar to raid0, but near_copies copies of each chunk is stored
43	* (each on a different drive). The starting device for each section is offset
44	* near_copies from the starting device of the previous section. Thus there
45	* are (near_copies * far_copies) of each chunk, and each is on a different
46	* drive. near_copies and far_copies must be at least one, and their product
47	* is at most raid_disks.
48	*
49	* If far_offset is true, then the far_copies are handled a bit differently.
50	* The copies are still in different stripes, but instead of being very far
51	* apart on disk, there are adjacent stripes.
52	*
53	* The far and offset algorithms are handled slightly differently if
54	* 'use_far_sets' is true. In this case, the array's devices are grouped into
55	* sets that are (near_copies * far_copies) in size. The far copied stripes
56	* are still shifted by 'near_copies' devices, but this shifting stays confined
57	* to the set rather than the entire array. This is done to improve the number
58	* of device combinations that can fail without causing the array to fail.
59	* Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
60	* on a device):
61	* A B C D A B C D E
62	* ... ...
63	* D A B C E A B C D
64	* Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
65	* [A B] [C D] [A B] [C D E]
66	* |...| |...| |...| | ... |
67	* [B A] [D C] [B A] [E C D]
68	*/
69
70	static void allow_barrier(struct r10conf *conf);
71	static void lower_barrier(struct r10conf *conf);
72	static int _enough(struct r10conf *conf, int previous, int ignore);
73	static int enough(struct r10conf *conf, int ignore);
74	static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
75	int *skipped);
76	static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
77	static void end_reshape_write(struct bio *bio);
78	static void end_reshape(struct r10conf *conf);
79
80	#include "raid1-10.c"
81
82	#define NULL_CMD
83	#define cmd_before(conf, cmd) \
84	do { \
85	write_sequnlock_irq(&(conf)->resync_lock); \
86	cmd; \
87	} while (0)
88	#define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock)
89
90	#define wait_event_barrier_cmd(conf, cond, cmd) \
91	wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \
92	cmd_after(conf))
93
94	#define wait_event_barrier(conf, cond) \
95	wait_event_barrier_cmd(conf, cond, NULL_CMD)
96
97	/*
98	* for resync bio, r10bio pointer can be retrieved from the per-bio
99	* 'struct resync_pages'.
100	*/
101	static inline struct r10bio *get_resync_r10bio(struct bio *bio)
102	{
103	return get_resync_pages(bio)->raid_bio;
104	}
105
106	static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
107	{
108	struct r10conf *conf = data;
109	int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]);
110
111	/* allocate a r10bio with room for raid_disks entries in the
112	* bios array */
113	return kzalloc(size, gfp_flags);
114	}
115
116	#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
117	/* amount of memory to reserve for resync requests */
118	#define RESYNC_WINDOW (1024*1024)
119	/* maximum number of concurrent requests, memory permitting */
120	#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
121	#define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW)
122	#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
123
124	/*
125	* When performing a resync, we need to read and compare, so
126	* we need as many pages are there are copies.
127	* When performing a recovery, we need 2 bios, one for read,
128	* one for write (we recover only one drive per r10buf)
129	*
130	*/
131	static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
132	{
133	struct r10conf *conf = data;
134	struct r10bio *r10_bio;
135	struct bio *bio;
136	int j;
137	int nalloc, nalloc_rp;
138	struct resync_pages *rps;
139
140	r10_bio = r10bio_pool_alloc(gfp_flags, data: conf);
141	if (!r10_bio)
142	return NULL;
143
144	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
145	test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
146	nalloc = conf->copies; /* resync */
147	else
148	nalloc = 2; /* recovery */
149
150	/* allocate once for all bios */
151	if (!conf->have_replacement)
152	nalloc_rp = nalloc;
153	else
154	nalloc_rp = nalloc * 2;
155	rps = kmalloc_array(nalloc_rp, sizeof(struct resync_pages), gfp_flags);
156	if (!rps)
157	goto out_free_r10bio;
158
159	/*
160	* Allocate bios.
161	*/
162	for (j = nalloc ; j-- ; ) {
163	bio = bio_kmalloc(RESYNC_PAGES, gfp_mask: gfp_flags);
164	if (!bio)
165	goto out_free_bio;
166	bio_init_inline(bio, NULL, RESYNC_PAGES, opf: 0);
167	r10_bio->devs[j].bio = bio;
168	if (!conf->have_replacement)
169	continue;
170	bio = bio_kmalloc(RESYNC_PAGES, gfp_mask: gfp_flags);
171	if (!bio)
172	goto out_free_bio;
173	bio_init_inline(bio, NULL, RESYNC_PAGES, opf: 0);
174	r10_bio->devs[j].repl_bio = bio;
175	}
176	/*
177	* Allocate RESYNC_PAGES data pages and attach them
178	* where needed.
179	*/
180	for (j = 0; j < nalloc; j++) {
181	struct bio *rbio = r10_bio->devs[j].repl_bio;
182	struct resync_pages *rp, *rp_repl;
183
184	rp = &rps[j];
185	if (rbio)
186	rp_repl = &rps[nalloc + j];
187
188	bio = r10_bio->devs[j].bio;
189
190	if (!j || test_bit(MD_RECOVERY_SYNC,
191	&conf->mddev->recovery)) {
192	if (resync_alloc_pages(rp, gfp_flags))
193	goto out_free_pages;
194	} else {
195	memcpy(rp, &rps[0], sizeof(*rp));
196	resync_get_all_pages(rp);
197	}
198
199	rp->raid_bio = r10_bio;
200	bio->bi_private = rp;
201	if (rbio) {
202	memcpy(rp_repl, rp, sizeof(*rp));
203	rbio->bi_private = rp_repl;
204	}
205	}
206
207	return r10_bio;
208
209	out_free_pages:
210	while (--j >= 0)
211	resync_free_pages(rp: &rps[j]);
212
213	j = 0;
214	out_free_bio:
215	for ( ; j < nalloc; j++) {
216	if (r10_bio->devs[j].bio)
217	bio_uninit(r10_bio->devs[j].bio);
218	kfree(objp: r10_bio->devs[j].bio);
219	if (r10_bio->devs[j].repl_bio)
220	bio_uninit(r10_bio->devs[j].repl_bio);
221	kfree(objp: r10_bio->devs[j].repl_bio);
222	}
223	kfree(objp: rps);
224	out_free_r10bio:
225	rbio_pool_free(rbio: r10_bio, data: conf);
226	return NULL;
227	}
228
229	static void r10buf_pool_free(void *__r10_bio, void *data)
230	{
231	struct r10conf *conf = data;
232	struct r10bio *r10bio = __r10_bio;
233	int j;
234	struct resync_pages *rp = NULL;
235
236	for (j = conf->copies; j--; ) {
237	struct bio *bio = r10bio->devs[j].bio;
238
239	if (bio) {
240	rp = get_resync_pages(bio);
241	resync_free_pages(rp);
242	bio_uninit(bio);
243	kfree(objp: bio);
244	}
245
246	bio = r10bio->devs[j].repl_bio;
247	if (bio) {
248	bio_uninit(bio);
249	kfree(objp: bio);
250	}
251	}
252
253	/* resync pages array stored in the 1st bio's .bi_private */
254	kfree(objp: rp);
255
256	rbio_pool_free(rbio: r10bio, data: conf);
257	}
258
259	static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
260	{
261	int i;
262
263	for (i = 0; i < conf->geo.raid_disks; i++) {
264	struct bio **bio = & r10_bio->devs[i].bio;
265	if (!BIO_SPECIAL(*bio))
266	bio_put(*bio);
267	*bio = NULL;
268	bio = &r10_bio->devs[i].repl_bio;
269	if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
270	bio_put(*bio);
271	*bio = NULL;
272	}
273	}
274
275	static void free_r10bio(struct r10bio *r10_bio)
276	{
277	struct r10conf *conf = r10_bio->mddev->private;
278
279	put_all_bios(conf, r10_bio);
280	mempool_free(element: r10_bio, pool: &conf->r10bio_pool);
281	}
282
283	static void put_buf(struct r10bio *r10_bio)
284	{
285	struct r10conf *conf = r10_bio->mddev->private;
286
287	mempool_free(element: r10_bio, pool: &conf->r10buf_pool);
288
289	lower_barrier(conf);
290	}
291
292	static void wake_up_barrier(struct r10conf *conf)
293	{
294	if (wq_has_sleeper(wq_head: &conf->wait_barrier))
295	wake_up(&conf->wait_barrier);
296	}
297
298	static void reschedule_retry(struct r10bio *r10_bio)
299	{
300	unsigned long flags;
301	struct mddev *mddev = r10_bio->mddev;
302	struct r10conf *conf = mddev->private;
303
304	spin_lock_irqsave(&conf->device_lock, flags);
305	list_add(new: &r10_bio->retry_list, head: &conf->retry_list);
306	conf->nr_queued ++;
307	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
308
309	/* wake up frozen array... */
310	wake_up(&conf->wait_barrier);
311
312	md_wakeup_thread(mddev->thread);
313	}
314
315	/*
316	* raid_end_bio_io() is called when we have finished servicing a mirrored
317	* operation and are ready to return a success/failure code to the buffer
318	* cache layer.
319	*/
320	static void raid_end_bio_io(struct r10bio *r10_bio)
321	{
322	struct bio *bio = r10_bio->master_bio;
323	struct r10conf *conf = r10_bio->mddev->private;
324
325	if (!test_and_set_bit(nr: R10BIO_Returned, addr: &r10_bio->state)) {
326	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
327	bio->bi_status = BLK_STS_IOERR;
328	bio_endio(bio);
329	}
330
331	/*
332	* Wake up any possible resync thread that waits for the device
333	* to go idle.
334	*/
335	allow_barrier(conf);
336
337	free_r10bio(r10_bio);
338	}
339
340	/*
341	* Update disk head position estimator based on IRQ completion info.
342	*/
343	static inline void update_head_pos(int slot, struct r10bio *r10_bio)
344	{
345	struct r10conf *conf = r10_bio->mddev->private;
346
347	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
348	r10_bio->devs[slot].addr + (r10_bio->sectors);
349	}
350
351	/*
352	* Find the disk number which triggered given bio
353	*/
354	static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
355	struct bio *bio, int *slotp, int *replp)
356	{
357	int slot;
358	int repl = 0;
359
360	for (slot = 0; slot < conf->geo.raid_disks; slot++) {
361	if (r10_bio->devs[slot].bio == bio)
362	break;
363	if (r10_bio->devs[slot].repl_bio == bio) {
364	repl = 1;
365	break;
366	}
367	}
368
369	update_head_pos(slot, r10_bio);
370
371	if (slotp)
372	*slotp = slot;
373	if (replp)
374	*replp = repl;
375	return r10_bio->devs[slot].devnum;
376	}
377
378	static void raid10_end_read_request(struct bio *bio)
379	{
380	int uptodate = !bio->bi_status;
381	struct r10bio *r10_bio = bio->bi_private;
382	int slot;
383	struct md_rdev *rdev;
384	struct r10conf *conf = r10_bio->mddev->private;
385
386	slot = r10_bio->read_slot;
387	rdev = r10_bio->devs[slot].rdev;
388	/*
389	* this branch is our 'one mirror IO has finished' event handler:
390	*/
391	update_head_pos(slot, r10_bio);
392
393	if (uptodate) {
394	/*
395	* Set R10BIO_Uptodate in our master bio, so that
396	* we will return a good error code to the higher
397	* levels even if IO on some other mirrored buffer fails.
398	*
399	* The 'master' represents the composite IO operation to
400	* user-side. So if something waits for IO, then it will
401	* wait for the 'master' bio.
402	*/
403	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
404	} else if (!raid1_should_handle_error(bio)) {
405	uptodate = 1;
406	} else {
407	/* If all other devices that store this block have
408	* failed, we want to return the error upwards rather
409	* than fail the last device. Here we redefine
410	* "uptodate" to mean "Don't want to retry"
411	*/
412	if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
413	ignore: rdev->raid_disk))
414	uptodate = 1;
415	}
416	if (uptodate) {
417	raid_end_bio_io(r10_bio);
418	rdev_dec_pending(rdev, mddev: conf->mddev);
419	} else {
420	/*
421	* oops, read error - keep the refcount on the rdev
422	*/
423	pr_err_ratelimited("md/raid10:%s: %pg: rescheduling sector %llu\n",
424	mdname(conf->mddev),
425	rdev->bdev,
426	(unsigned long long)r10_bio->sector);
427	set_bit(nr: R10BIO_ReadError, addr: &r10_bio->state);
428	reschedule_retry(r10_bio);
429	}
430	}
431
432	static void close_write(struct r10bio *r10_bio)
433	{
434	struct mddev *mddev = r10_bio->mddev;
435
436	md_write_end(mddev);
437	}
438
439	static void one_write_done(struct r10bio *r10_bio)
440	{
441	if (atomic_dec_and_test(v: &r10_bio->remaining)) {
442	if (test_bit(R10BIO_WriteError, &r10_bio->state))
443	reschedule_retry(r10_bio);
444	else {
445	close_write(r10_bio);
446	if (test_bit(R10BIO_MadeGood, &r10_bio->state))
447	reschedule_retry(r10_bio);
448	else
449	raid_end_bio_io(r10_bio);
450	}
451	}
452	}
453
454	static void raid10_end_write_request(struct bio *bio)
455	{
456	struct r10bio *r10_bio = bio->bi_private;
457	int dev;
458	int dec_rdev = 1;
459	struct r10conf *conf = r10_bio->mddev->private;
460	int slot, repl;
461	struct md_rdev *rdev = NULL;
462	struct bio *to_put = NULL;
463	bool ignore_error = !raid1_should_handle_error(bio) ||
464	(bio->bi_status && bio_op(bio) == REQ_OP_DISCARD);
465
466	dev = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
467
468	if (repl)
469	rdev = conf->mirrors[dev].replacement;
470	if (!rdev) {
471	smp_rmb();
472	repl = 0;
473	rdev = conf->mirrors[dev].rdev;
474	}
475	/*
476	* this branch is our 'one mirror IO has finished' event handler:
477	*/
478	if (bio->bi_status && !ignore_error) {
479	if (repl)
480	/* Never record new bad blocks to replacement,
481	* just fail it.
482	*/
483	md_error(mddev: rdev->mddev, rdev);
484	else {
485	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
486	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
487	set_bit(nr: MD_RECOVERY_NEEDED,
488	addr: &rdev->mddev->recovery);
489
490	dec_rdev = 0;
491	if (test_bit(FailFast, &rdev->flags) &&
492	(bio->bi_opf & MD_FAILFAST)) {
493	md_error(mddev: rdev->mddev, rdev);
494	}
495
496	/*
497	* When the device is faulty, it is not necessary to
498	* handle write error.
499	*/
500	if (!test_bit(Faulty, &rdev->flags))
501	set_bit(nr: R10BIO_WriteError, addr: &r10_bio->state);
502	else {
503	/* Fail the request */
504	r10_bio->devs[slot].bio = NULL;
505	to_put = bio;
506	dec_rdev = 1;
507	}
508	}
509	} else {
510	/*
511	* Set R10BIO_Uptodate in our master bio, so that
512	* we will return a good error code for to the higher
513	* levels even if IO on some other mirrored buffer fails.
514	*
515	* The 'master' represents the composite IO operation to
516	* user-side. So if something waits for IO, then it will
517	* wait for the 'master' bio.
518	*
519	* Do not set R10BIO_Uptodate if the current device is
520	* rebuilding or Faulty. This is because we cannot use
521	* such device for properly reading the data back (we could
522	* potentially use it, if the current write would have felt
523	* before rdev->recovery_offset, but for simplicity we don't
524	* check this here.
525	*/
526	if (test_bit(In_sync, &rdev->flags) &&
527	!test_bit(Faulty, &rdev->flags))
528	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
529
530	/* Maybe we can clear some bad blocks. */
531	if (rdev_has_badblock(rdev, s: r10_bio->devs[slot].addr,
532	sectors: r10_bio->sectors) &&
533	!ignore_error) {
534	bio_put(bio);
535	if (repl)
536	r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
537	else
538	r10_bio->devs[slot].bio = IO_MADE_GOOD;
539	dec_rdev = 0;
540	set_bit(nr: R10BIO_MadeGood, addr: &r10_bio->state);
541	}
542	}
543
544	/*
545	*
546	* Let's see if all mirrored write operations have finished
547	* already.
548	*/
549	one_write_done(r10_bio);
550	if (dec_rdev)
551	rdev_dec_pending(rdev, mddev: conf->mddev);
552	if (to_put)
553	bio_put(to_put);
554	}
555
556	/*
557	* RAID10 layout manager
558	* As well as the chunksize and raid_disks count, there are two
559	* parameters: near_copies and far_copies.
560	* near_copies * far_copies must be <= raid_disks.
561	* Normally one of these will be 1.
562	* If both are 1, we get raid0.
563	* If near_copies == raid_disks, we get raid1.
564	*
565	* Chunks are laid out in raid0 style with near_copies copies of the
566	* first chunk, followed by near_copies copies of the next chunk and
567	* so on.
568	* If far_copies > 1, then after 1/far_copies of the array has been assigned
569	* as described above, we start again with a device offset of near_copies.
570	* So we effectively have another copy of the whole array further down all
571	* the drives, but with blocks on different drives.
572	* With this layout, and block is never stored twice on the one device.
573	*
574	* raid10_find_phys finds the sector offset of a given virtual sector
575	* on each device that it is on.
576	*
577	* raid10_find_virt does the reverse mapping, from a device and a
578	* sector offset to a virtual address
579	*/
580
581	static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
582	{
583	int n,f;
584	sector_t sector;
585	sector_t chunk;
586	sector_t stripe;
587	int dev;
588	int slot = 0;
589	int last_far_set_start, last_far_set_size;
590
591	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
592	last_far_set_start *= geo->far_set_size;
593
594	last_far_set_size = geo->far_set_size;
595	last_far_set_size += (geo->raid_disks % geo->far_set_size);
596
597	/* now calculate first sector/dev */
598	chunk = r10bio->sector >> geo->chunk_shift;
599	sector = r10bio->sector & geo->chunk_mask;
600
601	chunk *= geo->near_copies;
602	stripe = chunk;
603	dev = sector_div(stripe, geo->raid_disks);
604	if (geo->far_offset)
605	stripe *= geo->far_copies;
606
607	sector += stripe << geo->chunk_shift;
608
609	/* and calculate all the others */
610	for (n = 0; n < geo->near_copies; n++) {
611	int d = dev;
612	int set;
613	sector_t s = sector;
614	r10bio->devs[slot].devnum = d;
615	r10bio->devs[slot].addr = s;
616	slot++;
617
618	for (f = 1; f < geo->far_copies; f++) {
619	set = d / geo->far_set_size;
620	d += geo->near_copies;
621
622	if ((geo->raid_disks % geo->far_set_size) &&
623	(d > last_far_set_start)) {
624	d -= last_far_set_start;
625	d %= last_far_set_size;
626	d += last_far_set_start;
627	} else {
628	d %= geo->far_set_size;
629	d += geo->far_set_size * set;
630	}
631	s += geo->stride;
632	r10bio->devs[slot].devnum = d;
633	r10bio->devs[slot].addr = s;
634	slot++;
635	}
636	dev++;
637	if (dev >= geo->raid_disks) {
638	dev = 0;
639	sector += (geo->chunk_mask + 1);
640	}
641	}
642	}
643
644	static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
645	{
646	struct geom *geo = &conf->geo;
647
648	if (conf->reshape_progress != MaxSector &&
649	((r10bio->sector >= conf->reshape_progress) !=
650	conf->mddev->reshape_backwards)) {
651	set_bit(nr: R10BIO_Previous, addr: &r10bio->state);
652	geo = &conf->prev;
653	} else
654	clear_bit(nr: R10BIO_Previous, addr: &r10bio->state);
655
656	__raid10_find_phys(geo, r10bio);
657	}
658
659	static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
660	{
661	sector_t offset, chunk, vchunk;
662	/* Never use conf->prev as this is only called during resync
663	* or recovery, so reshape isn't happening
664	*/
665	struct geom *geo = &conf->geo;
666	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
667	int far_set_size = geo->far_set_size;
668	int last_far_set_start;
669
670	if (geo->raid_disks % geo->far_set_size) {
671	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
672	last_far_set_start *= geo->far_set_size;
673
674	if (dev >= last_far_set_start) {
675	far_set_size = geo->far_set_size;
676	far_set_size += (geo->raid_disks % geo->far_set_size);
677	far_set_start = last_far_set_start;
678	}
679	}
680
681	offset = sector & geo->chunk_mask;
682	if (geo->far_offset) {
683	int fc;
684	chunk = sector >> geo->chunk_shift;
685	fc = sector_div(chunk, geo->far_copies);
686	dev -= fc * geo->near_copies;
687	if (dev < far_set_start)
688	dev += far_set_size;
689	} else {
690	while (sector >= geo->stride) {
691	sector -= geo->stride;
692	if (dev < (geo->near_copies + far_set_start))
693	dev += far_set_size - geo->near_copies;
694	else
695	dev -= geo->near_copies;
696	}
697	chunk = sector >> geo->chunk_shift;
698	}
699	vchunk = chunk * geo->raid_disks + dev;
700	sector_div(vchunk, geo->near_copies);
701	return (vchunk << geo->chunk_shift) + offset;
702	}
703
704	/*
705	* This routine returns the disk from which the requested read should
706	* be done. There is a per-array 'next expected sequential IO' sector
707	* number - if this matches on the next IO then we use the last disk.
708	* There is also a per-disk 'last know head position' sector that is
709	* maintained from IRQ contexts, both the normal and the resync IO
710	* completion handlers update this position correctly. If there is no
711	* perfect sequential match then we pick the disk whose head is closest.
712	*
713	* If there are 2 mirrors in the same 2 devices, performance degrades
714	* because position is mirror, not device based.
715	*
716	* The rdev for the device selected will have nr_pending incremented.
717	*/
718
719	/*
720	* FIXME: possibly should rethink readbalancing and do it differently
721	* depending on near_copies / far_copies geometry.
722	*/
723	static struct md_rdev *read_balance(struct r10conf *conf,
724	struct r10bio *r10_bio,
725	int *max_sectors)
726	{
727	const sector_t this_sector = r10_bio->sector;
728	int disk, slot;
729	int sectors = r10_bio->sectors;
730	int best_good_sectors;
731	sector_t new_distance, best_dist;
732	struct md_rdev *best_dist_rdev, *best_pending_rdev, *rdev = NULL;
733	int do_balance;
734	int best_dist_slot, best_pending_slot;
735	bool has_nonrot_disk = false;
736	unsigned int min_pending;
737	struct geom *geo = &conf->geo;
738
739	raid10_find_phys(conf, r10bio: r10_bio);
740	best_dist_slot = -1;
741	min_pending = UINT_MAX;
742	best_dist_rdev = NULL;
743	best_pending_rdev = NULL;
744	best_dist = MaxSector;
745	best_good_sectors = 0;
746	do_balance = 1;
747	clear_bit(nr: R10BIO_FailFast, addr: &r10_bio->state);
748
749	if (raid1_should_read_first(mddev: conf->mddev, this_sector, len: sectors))
750	do_balance = 0;
751
752	for (slot = 0; slot < conf->copies ; slot++) {
753	sector_t first_bad;
754	sector_t bad_sectors;
755	sector_t dev_sector;
756	unsigned int pending;
757	bool nonrot;
758
759	if (r10_bio->devs[slot].bio == IO_BLOCKED)
760	continue;
761	disk = r10_bio->devs[slot].devnum;
762	rdev = conf->mirrors[disk].replacement;
763	if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
764	r10_bio->devs[slot].addr + sectors >
765	rdev->recovery_offset)
766	rdev = conf->mirrors[disk].rdev;
767	if (rdev == NULL ||
768	test_bit(Faulty, &rdev->flags))
769	continue;
770	if (!test_bit(In_sync, &rdev->flags) &&
771	r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
772	continue;
773
774	dev_sector = r10_bio->devs[slot].addr;
775	if (is_badblock(rdev, s: dev_sector, sectors,
776	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
777	if (best_dist < MaxSector)
778	/* Already have a better slot */
779	continue;
780	if (first_bad <= dev_sector) {
781	/* Cannot read here. If this is the
782	* 'primary' device, then we must not read
783	* beyond 'bad_sectors' from another device.
784	*/
785	bad_sectors -= (dev_sector - first_bad);
786	if (!do_balance && sectors > bad_sectors)
787	sectors = bad_sectors;
788	if (best_good_sectors > sectors)
789	best_good_sectors = sectors;
790	} else {
791	sector_t good_sectors =
792	first_bad - dev_sector;
793	if (good_sectors > best_good_sectors) {
794	best_good_sectors = good_sectors;
795	best_dist_slot = slot;
796	best_dist_rdev = rdev;
797	}
798	if (!do_balance)
799	/* Must read from here */
800	break;
801	}
802	continue;
803	} else
804	best_good_sectors = sectors;
805
806	if (!do_balance)
807	break;
808
809	nonrot = bdev_nonrot(bdev: rdev->bdev);
810	has_nonrot_disk |= nonrot;
811	pending = atomic_read(v: &rdev->nr_pending);
812	if (min_pending > pending && nonrot) {
813	min_pending = pending;
814	best_pending_slot = slot;
815	best_pending_rdev = rdev;
816	}
817
818	if (best_dist_slot >= 0)
819	/* At least 2 disks to choose from so failfast is OK */
820	set_bit(nr: R10BIO_FailFast, addr: &r10_bio->state);
821	/* This optimisation is debatable, and completely destroys
822	* sequential read speed for 'far copies' arrays. So only
823	* keep it for 'near' arrays, and review those later.
824	*/
825	if (geo->near_copies > 1 && !pending)
826	new_distance = 0;
827
828	/* for far > 1 always use the lowest address */
829	else if (geo->far_copies > 1)
830	new_distance = r10_bio->devs[slot].addr;
831	else
832	new_distance = abs(r10_bio->devs[slot].addr -
833	conf->mirrors[disk].head_position);
834
835	if (new_distance < best_dist) {
836	best_dist = new_distance;
837	best_dist_slot = slot;
838	best_dist_rdev = rdev;
839	}
840	}
841	if (slot >= conf->copies) {
842	if (has_nonrot_disk) {
843	slot = best_pending_slot;
844	rdev = best_pending_rdev;
845	} else {
846	slot = best_dist_slot;
847	rdev = best_dist_rdev;
848	}
849	}
850
851	if (slot >= 0) {
852	atomic_inc(v: &rdev->nr_pending);
853	r10_bio->read_slot = slot;
854	} else
855	rdev = NULL;
856	*max_sectors = best_good_sectors;
857
858	return rdev;
859	}
860
861	static void flush_pending_writes(struct r10conf *conf)
862	{
863	/* Any writes that have been queued but are awaiting
864	* bitmap updates get flushed here.
865	*/
866	spin_lock_irq(lock: &conf->device_lock);
867
868	if (conf->pending_bio_list.head) {
869	struct blk_plug plug;
870	struct bio *bio;
871
872	bio = bio_list_get(bl: &conf->pending_bio_list);
873	spin_unlock_irq(lock: &conf->device_lock);
874
875	/*
876	* As this is called in a wait_event() loop (see freeze_array),
877	* current->state might be TASK_UNINTERRUPTIBLE which will
878	* cause a warning when we prepare to wait again. As it is
879	* rare that this path is taken, it is perfectly safe to force
880	* us to go around the wait_event() loop again, so the warning
881	* is a false-positive. Silence the warning by resetting
882	* thread state
883	*/
884	__set_current_state(TASK_RUNNING);
885
886	blk_start_plug(&plug);
887	raid1_prepare_flush_writes(mddev: conf->mddev);
888	wake_up(&conf->wait_barrier);
889
890	while (bio) { /* submit pending writes */
891	struct bio *next = bio->bi_next;
892
893	raid1_submit_write(bio);
894	bio = next;
895	cond_resched();
896	}
897	blk_finish_plug(&plug);
898	} else
899	spin_unlock_irq(lock: &conf->device_lock);
900	}
901
902	/* Barriers....
903	* Sometimes we need to suspend IO while we do something else,
904	* either some resync/recovery, or reconfigure the array.
905	* To do this we raise a 'barrier'.
906	* The 'barrier' is a counter that can be raised multiple times
907	* to count how many activities are happening which preclude
908	* normal IO.
909	* We can only raise the barrier if there is no pending IO.
910	* i.e. if nr_pending == 0.
911	* We choose only to raise the barrier if no-one is waiting for the
912	* barrier to go down. This means that as soon as an IO request
913	* is ready, no other operations which require a barrier will start
914	* until the IO request has had a chance.
915	*
916	* So: regular IO calls 'wait_barrier'. When that returns there
917	* is no backgroup IO happening, It must arrange to call
918	* allow_barrier when it has finished its IO.
919	* backgroup IO calls must call raise_barrier. Once that returns
920	* there is no normal IO happeing. It must arrange to call
921	* lower_barrier when the particular background IO completes.
922	*/
923
924	static void raise_barrier(struct r10conf *conf, int force)
925	{
926	write_seqlock_irq(sl: &conf->resync_lock);
927
928	if (WARN_ON_ONCE(force && !conf->barrier))
929	force = false;
930
931	/* Wait until no block IO is waiting (unless 'force') */
932	wait_event_barrier(conf, force || !conf->nr_waiting);
933
934	/* block any new IO from starting */
935	WRITE_ONCE(conf->barrier, conf->barrier + 1);
936
937	/* Now wait for all pending IO to complete */
938	wait_event_barrier(conf, !atomic_read(&conf->nr_pending) &&
939	conf->barrier < RESYNC_DEPTH);
940
941	write_sequnlock_irq(sl: &conf->resync_lock);
942	}
943
944	static void lower_barrier(struct r10conf *conf)
945	{
946	unsigned long flags;
947
948	write_seqlock_irqsave(&conf->resync_lock, flags);
949	WRITE_ONCE(conf->barrier, conf->barrier - 1);
950	write_sequnlock_irqrestore(sl: &conf->resync_lock, flags);
951	wake_up(&conf->wait_barrier);
952	}
953
954	static bool stop_waiting_barrier(struct r10conf *conf)
955	{
956	struct bio_list *bio_list = current->bio_list;
957	struct md_thread *thread;
958
959	/* barrier is dropped */
960	if (!conf->barrier)
961	return true;
962
963	/*
964	* If there are already pending requests (preventing the barrier from
965	* rising completely), and the pre-process bio queue isn't empty, then
966	* don't wait, as we need to empty that queue to get the nr_pending
967	* count down.
968	*/
969	if (atomic_read(v: &conf->nr_pending) && bio_list &&
970	(!bio_list_empty(bl: &bio_list[0]) || !bio_list_empty(bl: &bio_list[1])))
971	return true;
972
973	/* daemon thread must exist while handling io */
974	thread = rcu_dereference_protected(conf->mddev->thread, true);
975	/*
976	* move on if io is issued from raid10d(), nr_pending is not released
977	* from original io(see handle_read_error()). All raise barrier is
978	* blocked until this io is done.
979	*/
980	if (thread->tsk == current) {
981	WARN_ON_ONCE(atomic_read(&conf->nr_pending) == 0);
982	return true;
983	}
984
985	return false;
986	}
987
988	static bool wait_barrier_nolock(struct r10conf *conf)
989	{
990	unsigned int seq = read_seqbegin(sl: &conf->resync_lock);
991
992	if (READ_ONCE(conf->barrier))
993	return false;
994
995	atomic_inc(v: &conf->nr_pending);
996	if (!read_seqretry(sl: &conf->resync_lock, start: seq))
997	return true;
998
999	if (atomic_dec_and_test(v: &conf->nr_pending))
1000	wake_up_barrier(conf);
1001
1002	return false;
1003	}
1004
1005	static bool wait_barrier(struct r10conf *conf, bool nowait)
1006	{
1007	bool ret = true;
1008
1009	if (wait_barrier_nolock(conf))
1010	return true;
1011
1012	write_seqlock_irq(sl: &conf->resync_lock);
1013	if (conf->barrier) {
1014	/* Return false when nowait flag is set */
1015	if (nowait) {
1016	ret = false;
1017	} else {
1018	conf->nr_waiting++;
1019	mddev_add_trace_msg(conf->mddev, "raid10 wait barrier");
1020	wait_event_barrier(conf, stop_waiting_barrier(conf));
1021	conf->nr_waiting--;
1022	}
1023	if (!conf->nr_waiting)
1024	wake_up(&conf->wait_barrier);
1025	}
1026	/* Only increment nr_pending when we wait */
1027	if (ret)
1028	atomic_inc(v: &conf->nr_pending);
1029	write_sequnlock_irq(sl: &conf->resync_lock);
1030	return ret;
1031	}
1032
1033	static void allow_barrier(struct r10conf *conf)
1034	{
1035	if ((atomic_dec_and_test(v: &conf->nr_pending)) ||
1036	(conf->array_freeze_pending))
1037	wake_up_barrier(conf);
1038	}
1039
1040	static void freeze_array(struct r10conf *conf, int extra)
1041	{
1042	/* stop syncio and normal IO and wait for everything to
1043	* go quiet.
1044	* We increment barrier and nr_waiting, and then
1045	* wait until nr_pending match nr_queued+extra
1046	* This is called in the context of one normal IO request
1047	* that has failed. Thus any sync request that might be pending
1048	* will be blocked by nr_pending, and we need to wait for
1049	* pending IO requests to complete or be queued for re-try.
1050	* Thus the number queued (nr_queued) plus this request (extra)
1051	* must match the number of pending IOs (nr_pending) before
1052	* we continue.
1053	*/
1054	write_seqlock_irq(sl: &conf->resync_lock);
1055	conf->array_freeze_pending++;
1056	WRITE_ONCE(conf->barrier, conf->barrier + 1);
1057	conf->nr_waiting++;
1058	wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) ==
1059	conf->nr_queued + extra, flush_pending_writes(conf));
1060	conf->array_freeze_pending--;
1061	write_sequnlock_irq(sl: &conf->resync_lock);
1062	}
1063
1064	static void unfreeze_array(struct r10conf *conf)
1065	{
1066	/* reverse the effect of the freeze */
1067	write_seqlock_irq(sl: &conf->resync_lock);
1068	WRITE_ONCE(conf->barrier, conf->barrier - 1);
1069	conf->nr_waiting--;
1070	wake_up(&conf->wait_barrier);
1071	write_sequnlock_irq(sl: &conf->resync_lock);
1072	}
1073
1074	static sector_t choose_data_offset(struct r10bio *r10_bio,
1075	struct md_rdev *rdev)
1076	{
1077	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1078	test_bit(R10BIO_Previous, &r10_bio->state))
1079	return rdev->data_offset;
1080	else
1081	return rdev->new_data_offset;
1082	}
1083
1084	static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1085	{
1086	struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb, cb);
1087	struct mddev *mddev = plug->cb.data;
1088	struct r10conf *conf = mddev->private;
1089	struct bio *bio;
1090
1091	if (from_schedule) {
1092	spin_lock_irq(lock: &conf->device_lock);
1093	bio_list_merge(bl: &conf->pending_bio_list, bl2: &plug->pending);
1094	spin_unlock_irq(lock: &conf->device_lock);
1095	wake_up_barrier(conf);
1096	md_wakeup_thread(mddev->thread);
1097	kfree(objp: plug);
1098	return;
1099	}
1100
1101	/* we aren't scheduling, so we can do the write-out directly. */
1102	bio = bio_list_get(bl: &plug->pending);
1103	raid1_prepare_flush_writes(mddev);
1104	wake_up_barrier(conf);
1105
1106	while (bio) { /* submit pending writes */
1107	struct bio *next = bio->bi_next;
1108
1109	raid1_submit_write(bio);
1110	bio = next;
1111	cond_resched();
1112	}
1113	kfree(objp: plug);
1114	}
1115
1116	/*
1117	* 1. Register the new request and wait if the reconstruction thread has put
1118	* up a bar for new requests. Continue immediately if no resync is active
1119	* currently.
1120	* 2. If IO spans the reshape position. Need to wait for reshape to pass.
1121	*/
1122	static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf,
1123	struct bio *bio, sector_t sectors)
1124	{
1125	/* Bail out if REQ_NOWAIT is set for the bio */
1126	if (!wait_barrier(conf, nowait: bio->bi_opf & REQ_NOWAIT)) {
1127	bio_wouldblock_error(bio);
1128	return false;
1129	}
1130	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1131	bio->bi_iter.bi_sector < conf->reshape_progress &&
1132	bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1133	allow_barrier(conf);
1134	if (bio->bi_opf & REQ_NOWAIT) {
1135	bio_wouldblock_error(bio);
1136	return false;
1137	}
1138	mddev_add_trace_msg(conf->mddev, "raid10 wait reshape");
1139	wait_event(conf->wait_barrier,
1140	conf->reshape_progress <= bio->bi_iter.bi_sector ||
1141	conf->reshape_progress >= bio->bi_iter.bi_sector +
1142	sectors);
1143	wait_barrier(conf, nowait: false);
1144	}
1145	return true;
1146	}
1147
1148	static void raid10_read_request(struct mddev *mddev, struct bio *bio,
1149	struct r10bio *r10_bio, bool io_accounting)
1150	{
1151	struct r10conf *conf = mddev->private;
1152	struct bio *read_bio;
1153	int max_sectors;
1154	struct md_rdev *rdev;
1155	char b[BDEVNAME_SIZE];
1156	int slot = r10_bio->read_slot;
1157	struct md_rdev *err_rdev = NULL;
1158	gfp_t gfp = GFP_NOIO;
1159
1160	if (slot >= 0 && r10_bio->devs[slot].rdev) {
1161	/*
1162	* This is an error retry, but we cannot
1163	* safely dereference the rdev in the r10_bio,
1164	* we must use the one in conf.
1165	* If it has already been disconnected (unlikely)
1166	* we lose the device name in error messages.
1167	*/
1168	int disk;
1169	/*
1170	* As we are blocking raid10, it is a little safer to
1171	* use __GFP_HIGH.
1172	*/
1173	gfp = GFP_NOIO | __GFP_HIGH;
1174
1175	disk = r10_bio->devs[slot].devnum;
1176	err_rdev = conf->mirrors[disk].rdev;
1177	if (err_rdev)
1178	snprintf(buf: b, size: sizeof(b), fmt: "%pg", err_rdev->bdev);
1179	else {
1180	strcpy(p: b, q: "???");
1181	/* This never gets dereferenced */
1182	err_rdev = r10_bio->devs[slot].rdev;
1183	}
1184	}
1185
1186	if (!regular_request_wait(mddev, conf, bio, sectors: r10_bio->sectors)) {
1187	raid_end_bio_io(r10_bio);
1188	return;
1189	}
1190
1191	rdev = read_balance(conf, r10_bio, max_sectors: &max_sectors);
1192	if (!rdev) {
1193	if (err_rdev) {
1194	pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
1195	mdname(mddev), b,
1196	(unsigned long long)r10_bio->sector);
1197	}
1198	raid_end_bio_io(r10_bio);
1199	return;
1200	}
1201	if (err_rdev)
1202	pr_err_ratelimited("md/raid10:%s: %pg: redirecting sector %llu to another mirror\n",
1203	mdname(mddev),
1204	rdev->bdev,
1205	(unsigned long long)r10_bio->sector);
1206	if (max_sectors < bio_sectors(bio)) {
1207	allow_barrier(conf);
1208	bio = bio_submit_split_bioset(bio, split_sectors: max_sectors,
1209	bs: &conf->bio_split);
1210	wait_barrier(conf, nowait: false);
1211	if (!bio) {
1212	set_bit(nr: R10BIO_Returned, addr: &r10_bio->state);
1213	goto err_handle;
1214	}
1215
1216	r10_bio->master_bio = bio;
1217	r10_bio->sectors = max_sectors;
1218	}
1219	slot = r10_bio->read_slot;
1220
1221	if (io_accounting) {
1222	md_account_bio(mddev, bio: &bio);
1223	r10_bio->master_bio = bio;
1224	}
1225	read_bio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, gfp, bs: &mddev->bio_set);
1226	read_bio->bi_opf &= ~REQ_NOWAIT;
1227
1228	r10_bio->devs[slot].bio = read_bio;
1229	r10_bio->devs[slot].rdev = rdev;
1230
1231	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
1232	choose_data_offset(r10_bio, rdev);
1233	read_bio->bi_end_io = raid10_end_read_request;
1234	if (test_bit(FailFast, &rdev->flags) &&
1235	test_bit(R10BIO_FailFast, &r10_bio->state))
1236	read_bio->bi_opf |= MD_FAILFAST;
1237	read_bio->bi_private = r10_bio;
1238	mddev_trace_remap(mddev, bio: read_bio, sector: r10_bio->sector);
1239	submit_bio_noacct(bio: read_bio);
1240	return;
1241	err_handle:
1242	atomic_dec(v: &rdev->nr_pending);
1243	raid_end_bio_io(r10_bio);
1244	}
1245
1246	static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
1247	struct bio *bio, bool replacement,
1248	int n_copy)
1249	{
1250	unsigned long flags;
1251	struct r10conf *conf = mddev->private;
1252	struct md_rdev *rdev;
1253	int devnum = r10_bio->devs[n_copy].devnum;
1254	struct bio *mbio;
1255
1256	rdev = replacement ? conf->mirrors[devnum].replacement :
1257	conf->mirrors[devnum].rdev;
1258
1259	mbio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, GFP_NOIO, bs: &mddev->bio_set);
1260	mbio->bi_opf &= ~REQ_NOWAIT;
1261	if (replacement)
1262	r10_bio->devs[n_copy].repl_bio = mbio;
1263	else
1264	r10_bio->devs[n_copy].bio = mbio;
1265
1266	mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr +
1267	choose_data_offset(r10_bio, rdev));
1268	mbio->bi_end_io = raid10_end_write_request;
1269	if (!replacement && test_bit(FailFast,
1270	&conf->mirrors[devnum].rdev->flags)
1271	&& enough(conf, ignore: devnum))
1272	mbio->bi_opf |= MD_FAILFAST;
1273	mbio->bi_private = r10_bio;
1274	mddev_trace_remap(mddev, bio: mbio, sector: r10_bio->sector);
1275	/* flush_pending_writes() needs access to the rdev so...*/
1276	mbio->bi_bdev = (void *)rdev;
1277
1278	atomic_inc(v: &r10_bio->remaining);
1279
1280	if (!raid1_add_bio_to_plug(mddev, bio: mbio, unplug: raid10_unplug, copies: conf->copies)) {
1281	spin_lock_irqsave(&conf->device_lock, flags);
1282	bio_list_add(bl: &conf->pending_bio_list, bio: mbio);
1283	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
1284	md_wakeup_thread(mddev->thread);
1285	}
1286	}
1287
1288	static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)
1289	{
1290	struct r10conf *conf = mddev->private;
1291	struct md_rdev *blocked_rdev;
1292	int i;
1293
1294	retry_wait:
1295	blocked_rdev = NULL;
1296	for (i = 0; i < conf->copies; i++) {
1297	struct md_rdev *rdev, *rrdev;
1298
1299	rdev = conf->mirrors[i].rdev;
1300	if (rdev) {
1301	sector_t dev_sector = r10_bio->devs[i].addr;
1302
1303	/*
1304	* Discard request doesn't care the write result
1305	* so it doesn't need to wait blocked disk here.
1306	*/
1307	if (test_bit(WriteErrorSeen, &rdev->flags) &&
1308	r10_bio->sectors &&
1309	rdev_has_badblock(rdev, s: dev_sector,
1310	sectors: r10_bio->sectors) < 0)
1311	/*
1312	* Mustn't write here until the bad
1313	* block is acknowledged
1314	*/
1315	set_bit(nr: BlockedBadBlocks, addr: &rdev->flags);
1316
1317	if (rdev_blocked(rdev)) {
1318	blocked_rdev = rdev;
1319	atomic_inc(v: &rdev->nr_pending);
1320	break;
1321	}
1322	}
1323
1324	rrdev = conf->mirrors[i].replacement;
1325	if (rrdev && rdev_blocked(rdev: rrdev)) {
1326	atomic_inc(v: &rrdev->nr_pending);
1327	blocked_rdev = rrdev;
1328	break;
1329	}
1330	}
1331
1332	if (unlikely(blocked_rdev)) {
1333	/* Have to wait for this device to get unblocked, then retry */
1334	allow_barrier(conf);
1335	mddev_add_trace_msg(conf->mddev,
1336	"raid10 %s wait rdev %d blocked",
1337	__func__, blocked_rdev->raid_disk);
1338	md_wait_for_blocked_rdev(rdev: blocked_rdev, mddev);
1339	wait_barrier(conf, nowait: false);
1340	goto retry_wait;
1341	}
1342	}
1343
1344	static void raid10_write_request(struct mddev *mddev, struct bio *bio,
1345	struct r10bio *r10_bio)
1346	{
1347	struct r10conf *conf = mddev->private;
1348	int i, k;
1349	sector_t sectors;
1350	int max_sectors;
1351
1352	if ((mddev_is_clustered(mddev) &&
1353	mddev->cluster_ops->area_resyncing(mddev, WRITE,
1354	bio->bi_iter.bi_sector,
1355	bio_end_sector(bio)))) {
1356	DEFINE_WAIT(w);
1357	/* Bail out if REQ_NOWAIT is set for the bio */
1358	if (bio->bi_opf & REQ_NOWAIT) {
1359	bio_wouldblock_error(bio);
1360	return;
1361	}
1362	for (;;) {
1363	prepare_to_wait(wq_head: &conf->wait_barrier,
1364	wq_entry: &w, TASK_IDLE);
1365	if (!mddev->cluster_ops->area_resyncing(mddev, WRITE,
1366	bio->bi_iter.bi_sector, bio_end_sector(bio)))
1367	break;
1368	schedule();
1369	}
1370	finish_wait(wq_head: &conf->wait_barrier, wq_entry: &w);
1371	}
1372
1373	sectors = r10_bio->sectors;
1374	if (!regular_request_wait(mddev, conf, bio, sectors)) {
1375	raid_end_bio_io(r10_bio);
1376	return;
1377	}
1378
1379	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1380	(mddev->reshape_backwards
1381	? (bio->bi_iter.bi_sector < conf->reshape_safe &&
1382	bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
1383	: (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
1384	bio->bi_iter.bi_sector < conf->reshape_progress))) {
1385	/* Need to update reshape_position in metadata */
1386	mddev->reshape_position = conf->reshape_progress;
1387	set_mask_bits(&mddev->sb_flags, 0,
1388	BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1389	md_wakeup_thread(mddev->thread);
1390	if (bio->bi_opf & REQ_NOWAIT) {
1391	allow_barrier(conf);
1392	bio_wouldblock_error(bio);
1393	return;
1394	}
1395	mddev_add_trace_msg(conf->mddev,
1396	"raid10 wait reshape metadata");
1397	wait_event(mddev->sb_wait,
1398	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
1399
1400	conf->reshape_safe = mddev->reshape_position;
1401	}
1402
1403	/* first select target devices under rcu_lock and
1404	* inc refcount on their rdev. Record them by setting
1405	* bios[x] to bio
1406	* If there are known/acknowledged bad blocks on any device
1407	* on which we have seen a write error, we want to avoid
1408	* writing to those blocks. This potentially requires several
1409	* writes to write around the bad blocks. Each set of writes
1410	* gets its own r10_bio with a set of bios attached.
1411	*/
1412
1413	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1414	raid10_find_phys(conf, r10bio: r10_bio);
1415
1416	wait_blocked_dev(mddev, r10_bio);
1417
1418	max_sectors = r10_bio->sectors;
1419
1420	for (i = 0; i < conf->copies; i++) {
1421	int d = r10_bio->devs[i].devnum;
1422	struct md_rdev *rdev, *rrdev;
1423
1424	rdev = conf->mirrors[d].rdev;
1425	rrdev = conf->mirrors[d].replacement;
1426	if (rdev && (test_bit(Faulty, &rdev->flags)))
1427	rdev = NULL;
1428	if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1429	rrdev = NULL;
1430
1431	r10_bio->devs[i].bio = NULL;
1432	r10_bio->devs[i].repl_bio = NULL;
1433
1434	if (!rdev && !rrdev)
1435	continue;
1436	if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1437	sector_t first_bad;
1438	sector_t dev_sector = r10_bio->devs[i].addr;
1439	sector_t bad_sectors;
1440	int is_bad;
1441
1442	is_bad = is_badblock(rdev, s: dev_sector, sectors: max_sectors,
1443	first_bad: &first_bad, bad_sectors: &bad_sectors);
1444	if (is_bad && first_bad <= dev_sector) {
1445	/* Cannot write here at all */
1446	bad_sectors -= (dev_sector - first_bad);
1447	if (bad_sectors < max_sectors)
1448	/* Mustn't write more than bad_sectors
1449	* to other devices yet
1450	*/
1451	max_sectors = bad_sectors;
1452	continue;
1453	}
1454	if (is_bad) {
1455	int good_sectors;
1456
1457	/*
1458	* We cannot atomically write this, so just
1459	* error in that case. It could be possible to
1460	* atomically write other mirrors, but the
1461	* complexity of supporting that is not worth
1462	* the benefit.
1463	*/
1464	if (bio->bi_opf & REQ_ATOMIC)
1465	goto err_handle;
1466
1467	good_sectors = first_bad - dev_sector;
1468	if (good_sectors < max_sectors)
1469	max_sectors = good_sectors;
1470	}
1471	}
1472	if (rdev) {
1473	r10_bio->devs[i].bio = bio;
1474	atomic_inc(v: &rdev->nr_pending);
1475	}
1476	if (rrdev) {
1477	r10_bio->devs[i].repl_bio = bio;
1478	atomic_inc(v: &rrdev->nr_pending);
1479	}
1480	}
1481
1482	if (max_sectors < r10_bio->sectors)
1483	r10_bio->sectors = max_sectors;
1484
1485	if (r10_bio->sectors < bio_sectors(bio)) {
1486	allow_barrier(conf);
1487	bio = bio_submit_split_bioset(bio, split_sectors: r10_bio->sectors,
1488	bs: &conf->bio_split);
1489	wait_barrier(conf, nowait: false);
1490	if (!bio) {
1491	set_bit(nr: R10BIO_Returned, addr: &r10_bio->state);
1492	goto err_handle;
1493	}
1494
1495	r10_bio->master_bio = bio;
1496	}
1497
1498	md_account_bio(mddev, bio: &bio);
1499	r10_bio->master_bio = bio;
1500	atomic_set(v: &r10_bio->remaining, i: 1);
1501
1502	for (i = 0; i < conf->copies; i++) {
1503	if (r10_bio->devs[i].bio)
1504	raid10_write_one_disk(mddev, r10_bio, bio, replacement: false, n_copy: i);
1505	if (r10_bio->devs[i].repl_bio)
1506	raid10_write_one_disk(mddev, r10_bio, bio, replacement: true, n_copy: i);
1507	}
1508	one_write_done(r10_bio);
1509	return;
1510	err_handle:
1511	for (k = 0; k < i; k++) {
1512	int d = r10_bio->devs[k].devnum;
1513	struct md_rdev *rdev = conf->mirrors[d].rdev;
1514	struct md_rdev *rrdev = conf->mirrors[d].replacement;
1515
1516	if (r10_bio->devs[k].bio) {
1517	rdev_dec_pending(rdev, mddev);
1518	r10_bio->devs[k].bio = NULL;
1519	}
1520	if (r10_bio->devs[k].repl_bio) {
1521	rdev_dec_pending(rdev: rrdev, mddev);
1522	r10_bio->devs[k].repl_bio = NULL;
1523	}
1524	}
1525
1526	raid_end_bio_io(r10_bio);
1527	}
1528
1529	static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
1530	{
1531	struct r10conf *conf = mddev->private;
1532	struct r10bio *r10_bio;
1533
1534	r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);
1535
1536	r10_bio->master_bio = bio;
1537	r10_bio->sectors = sectors;
1538
1539	r10_bio->mddev = mddev;
1540	r10_bio->sector = bio->bi_iter.bi_sector;
1541	r10_bio->state = 0;
1542	r10_bio->read_slot = -1;
1543	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) *
1544	conf->geo.raid_disks);
1545
1546	if (bio_data_dir(bio) == READ)
1547	raid10_read_request(mddev, bio, r10_bio, io_accounting: true);
1548	else
1549	raid10_write_request(mddev, bio, r10_bio);
1550	}
1551
1552	static void raid_end_discard_bio(struct r10bio *r10bio)
1553	{
1554	struct r10conf *conf = r10bio->mddev->private;
1555	struct r10bio *first_r10bio;
1556
1557	while (atomic_dec_and_test(v: &r10bio->remaining)) {
1558
1559	allow_barrier(conf);
1560
1561	if (!test_bit(R10BIO_Discard, &r10bio->state)) {
1562	first_r10bio = (struct r10bio *)r10bio->master_bio;
1563	free_r10bio(r10_bio: r10bio);
1564	r10bio = first_r10bio;
1565	} else {
1566	md_write_end(mddev: r10bio->mddev);
1567	bio_endio(r10bio->master_bio);
1568	free_r10bio(r10_bio: r10bio);
1569	break;
1570	}
1571	}
1572	}
1573
1574	static void raid10_end_discard_request(struct bio *bio)
1575	{
1576	struct r10bio *r10_bio = bio->bi_private;
1577	struct r10conf *conf = r10_bio->mddev->private;
1578	struct md_rdev *rdev = NULL;
1579	int dev;
1580	int slot, repl;
1581
1582	/*
1583	* We don't care the return value of discard bio
1584	*/
1585	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
1586	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
1587
1588	dev = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
1589	rdev = repl ? conf->mirrors[dev].replacement :
1590	conf->mirrors[dev].rdev;
1591
1592	raid_end_discard_bio(r10bio: r10_bio);
1593	rdev_dec_pending(rdev, mddev: conf->mddev);
1594	}
1595
1596	/*
1597	* There are some limitations to handle discard bio
1598	* 1st, the discard size is bigger than stripe_size*2.
1599	* 2st, if the discard bio spans reshape progress, we use the old way to
1600	* handle discard bio
1601	*/
1602	static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
1603	{
1604	struct r10conf *conf = mddev->private;
1605	struct geom *geo = &conf->geo;
1606	int far_copies = geo->far_copies;
1607	bool first_copy = true;
1608	struct r10bio *r10_bio, *first_r10bio;
1609	struct bio *split;
1610	int disk;
1611	sector_t chunk;
1612	unsigned int stripe_size;
1613	unsigned int stripe_data_disks;
1614	sector_t split_size;
1615	sector_t bio_start, bio_end;
1616	sector_t first_stripe_index, last_stripe_index;
1617	sector_t start_disk_offset;
1618	unsigned int start_disk_index;
1619	sector_t end_disk_offset;
1620	unsigned int end_disk_index;
1621	unsigned int remainder;
1622
1623	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1624	return -EAGAIN;
1625
1626	if (!wait_barrier(conf, nowait: bio->bi_opf & REQ_NOWAIT)) {
1627	bio_wouldblock_error(bio);
1628	return 0;
1629	}
1630
1631	/*
1632	* Check reshape again to avoid reshape happens after checking
1633	* MD_RECOVERY_RESHAPE and before wait_barrier
1634	*/
1635	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1636	goto out;
1637
1638	if (geo->near_copies)
1639	stripe_data_disks = geo->raid_disks / geo->near_copies +
1640	geo->raid_disks % geo->near_copies;
1641	else
1642	stripe_data_disks = geo->raid_disks;
1643
1644	stripe_size = stripe_data_disks << geo->chunk_shift;
1645
1646	bio_start = bio->bi_iter.bi_sector;
1647	bio_end = bio_end_sector(bio);
1648
1649	/*
1650	* Maybe one discard bio is smaller than strip size or across one
1651	* stripe and discard region is larger than one stripe size. For far
1652	* offset layout, if the discard region is not aligned with stripe
1653	* size, there is hole when we submit discard bio to member disk.
1654	* For simplicity, we only handle discard bio which discard region
1655	* is bigger than stripe_size * 2
1656	*/
1657	if (bio_sectors(bio) < stripe_size*2)
1658	goto out;
1659
1660	/*
1661	* Keep bio aligned with strip size.
1662	*/
1663	div_u64_rem(dividend: bio_start, divisor: stripe_size, remainder: &remainder);
1664	if (remainder) {
1665	split_size = stripe_size - remainder;
1666	split = bio_split(bio, sectors: split_size, GFP_NOIO, bs: &conf->bio_split);
1667	if (IS_ERR(ptr: split)) {
1668	bio->bi_status = errno_to_blk_status(errno: PTR_ERR(ptr: split));
1669	bio_endio(bio);
1670	return 0;
1671	}
1672
1673	bio_chain(split, bio);
1674	trace_block_split(bio: split, new_sector: bio->bi_iter.bi_sector);
1675	allow_barrier(conf);
1676	/* Resend the fist split part */
1677	submit_bio_noacct(bio: split);
1678	wait_barrier(conf, nowait: false);
1679	}
1680	div_u64_rem(dividend: bio_end, divisor: stripe_size, remainder: &remainder);
1681	if (remainder) {
1682	split_size = bio_sectors(bio) - remainder;
1683	split = bio_split(bio, sectors: split_size, GFP_NOIO, bs: &conf->bio_split);
1684	if (IS_ERR(ptr: split)) {
1685	bio->bi_status = errno_to_blk_status(errno: PTR_ERR(ptr: split));
1686	bio_endio(bio);
1687	return 0;
1688	}
1689
1690	bio_chain(split, bio);
1691	trace_block_split(bio: split, new_sector: bio->bi_iter.bi_sector);
1692	allow_barrier(conf);
1693	/* Resend the second split part */
1694	submit_bio_noacct(bio);
1695	bio = split;
1696	wait_barrier(conf, nowait: false);
1697	}
1698
1699	bio_start = bio->bi_iter.bi_sector;
1700	bio_end = bio_end_sector(bio);
1701
1702	/*
1703	* Raid10 uses chunk as the unit to store data. It's similar like raid0.
1704	* One stripe contains the chunks from all member disk (one chunk from
1705	* one disk at the same HBA address). For layout detail, see 'man md 4'
1706	*/
1707	chunk = bio_start >> geo->chunk_shift;
1708	chunk *= geo->near_copies;
1709	first_stripe_index = chunk;
1710	start_disk_index = sector_div(first_stripe_index, geo->raid_disks);
1711	if (geo->far_offset)
1712	first_stripe_index *= geo->far_copies;
1713	start_disk_offset = (bio_start & geo->chunk_mask) +
1714	(first_stripe_index << geo->chunk_shift);
1715
1716	chunk = bio_end >> geo->chunk_shift;
1717	chunk *= geo->near_copies;
1718	last_stripe_index = chunk;
1719	end_disk_index = sector_div(last_stripe_index, geo->raid_disks);
1720	if (geo->far_offset)
1721	last_stripe_index *= geo->far_copies;
1722	end_disk_offset = (bio_end & geo->chunk_mask) +
1723	(last_stripe_index << geo->chunk_shift);
1724
1725	retry_discard:
1726	r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);
1727	r10_bio->mddev = mddev;
1728	r10_bio->state = 0;
1729	r10_bio->sectors = 0;
1730	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks);
1731	wait_blocked_dev(mddev, r10_bio);
1732
1733	/*
1734	* For far layout it needs more than one r10bio to cover all regions.
1735	* Inspired by raid10_sync_request, we can use the first r10bio->master_bio
1736	* to record the discard bio. Other r10bio->master_bio record the first
1737	* r10bio. The first r10bio only release after all other r10bios finish.
1738	* The discard bio returns only first r10bio finishes
1739	*/
1740	if (first_copy) {
1741	md_account_bio(mddev, bio: &bio);
1742	r10_bio->master_bio = bio;
1743	set_bit(nr: R10BIO_Discard, addr: &r10_bio->state);
1744	first_copy = false;
1745	first_r10bio = r10_bio;
1746	} else
1747	r10_bio->master_bio = (struct bio *)first_r10bio;
1748
1749	/*
1750	* first select target devices under rcu_lock and
1751	* inc refcount on their rdev. Record them by setting
1752	* bios[x] to bio
1753	*/
1754	for (disk = 0; disk < geo->raid_disks; disk++) {
1755	struct md_rdev *rdev, *rrdev;
1756
1757	rdev = conf->mirrors[disk].rdev;
1758	rrdev = conf->mirrors[disk].replacement;
1759	r10_bio->devs[disk].bio = NULL;
1760	r10_bio->devs[disk].repl_bio = NULL;
1761
1762	if (rdev && (test_bit(Faulty, &rdev->flags)))
1763	rdev = NULL;
1764	if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1765	rrdev = NULL;
1766	if (!rdev && !rrdev)
1767	continue;
1768
1769	if (rdev) {
1770	r10_bio->devs[disk].bio = bio;
1771	atomic_inc(v: &rdev->nr_pending);
1772	}
1773	if (rrdev) {
1774	r10_bio->devs[disk].repl_bio = bio;
1775	atomic_inc(v: &rrdev->nr_pending);
1776	}
1777	}
1778
1779	atomic_set(v: &r10_bio->remaining, i: 1);
1780	for (disk = 0; disk < geo->raid_disks; disk++) {
1781	sector_t dev_start, dev_end;
1782	struct bio *mbio, *rbio = NULL;
1783
1784	/*
1785	* Now start to calculate the start and end address for each disk.
1786	* The space between dev_start and dev_end is the discard region.
1787	*
1788	* For dev_start, it needs to consider three conditions:
1789	* 1st, the disk is before start_disk, you can imagine the disk in
1790	* the next stripe. So the dev_start is the start address of next
1791	* stripe.
1792	* 2st, the disk is after start_disk, it means the disk is at the
1793	* same stripe of first disk
1794	* 3st, the first disk itself, we can use start_disk_offset directly
1795	*/
1796	if (disk < start_disk_index)
1797	dev_start = (first_stripe_index + 1) * mddev->chunk_sectors;
1798	else if (disk > start_disk_index)
1799	dev_start = first_stripe_index * mddev->chunk_sectors;
1800	else
1801	dev_start = start_disk_offset;
1802
1803	if (disk < end_disk_index)
1804	dev_end = (last_stripe_index + 1) * mddev->chunk_sectors;
1805	else if (disk > end_disk_index)
1806	dev_end = last_stripe_index * mddev->chunk_sectors;
1807	else
1808	dev_end = end_disk_offset;
1809
1810	/*
1811	* It only handles discard bio which size is >= stripe size, so
1812	* dev_end > dev_start all the time.
1813	* It doesn't need to use rcu lock to get rdev here. We already
1814	* add rdev->nr_pending in the first loop.
1815	*/
1816	if (r10_bio->devs[disk].bio) {
1817	struct md_rdev *rdev = conf->mirrors[disk].rdev;
1818	mbio = bio_alloc_clone(bdev: bio->bi_bdev, bio_src: bio, GFP_NOIO,
1819	bs: &mddev->bio_set);
1820	mbio->bi_end_io = raid10_end_discard_request;
1821	mbio->bi_private = r10_bio;
1822	r10_bio->devs[disk].bio = mbio;
1823	r10_bio->devs[disk].devnum = disk;
1824	atomic_inc(v: &r10_bio->remaining);
1825	md_submit_discard_bio(mddev, rdev, bio: mbio,
1826	start: dev_start + choose_data_offset(r10_bio, rdev),
1827	size: dev_end - dev_start);
1828	bio_endio(mbio);
1829	}
1830	if (r10_bio->devs[disk].repl_bio) {
1831	struct md_rdev *rrdev = conf->mirrors[disk].replacement;
1832	rbio = bio_alloc_clone(bdev: bio->bi_bdev, bio_src: bio, GFP_NOIO,
1833	bs: &mddev->bio_set);
1834	rbio->bi_end_io = raid10_end_discard_request;
1835	rbio->bi_private = r10_bio;
1836	r10_bio->devs[disk].repl_bio = rbio;
1837	r10_bio->devs[disk].devnum = disk;
1838	atomic_inc(v: &r10_bio->remaining);
1839	md_submit_discard_bio(mddev, rdev: rrdev, bio: rbio,
1840	start: dev_start + choose_data_offset(r10_bio, rdev: rrdev),
1841	size: dev_end - dev_start);
1842	bio_endio(rbio);
1843	}
1844	}
1845
1846	if (!geo->far_offset && --far_copies) {
1847	first_stripe_index += geo->stride >> geo->chunk_shift;
1848	start_disk_offset += geo->stride;
1849	last_stripe_index += geo->stride >> geo->chunk_shift;
1850	end_disk_offset += geo->stride;
1851	atomic_inc(v: &first_r10bio->remaining);
1852	raid_end_discard_bio(r10bio: r10_bio);
1853	wait_barrier(conf, nowait: false);
1854	goto retry_discard;
1855	}
1856
1857	raid_end_discard_bio(r10bio: r10_bio);
1858
1859	return 0;
1860	out:
1861	allow_barrier(conf);
1862	return -EAGAIN;
1863	}
1864
1865	static bool raid10_make_request(struct mddev *mddev, struct bio *bio)
1866	{
1867	struct r10conf *conf = mddev->private;
1868	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1869	int chunk_sects = chunk_mask + 1;
1870	int sectors = bio_sectors(bio);
1871
1872	if (unlikely(bio->bi_opf & REQ_PREFLUSH)
1873	&& md_flush_request(mddev, bio))
1874	return true;
1875
1876	md_write_start(mddev, bi: bio);
1877
1878	if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
1879	if (!raid10_handle_discard(mddev, bio))
1880	return true;
1881
1882	/*
1883	* If this request crosses a chunk boundary, we need to split
1884	* it.
1885	*/
1886	if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
1887	sectors > chunk_sects
1888	&& (conf->geo.near_copies < conf->geo.raid_disks
1889	|| conf->prev.near_copies <
1890	conf->prev.raid_disks)))
1891	sectors = chunk_sects -
1892	(bio->bi_iter.bi_sector &
1893	(chunk_sects - 1));
1894	__make_request(mddev, bio, sectors);
1895
1896	/* In case raid10d snuck in to freeze_array */
1897	wake_up_barrier(conf);
1898	return true;
1899	}
1900
1901	static void raid10_status(struct seq_file *seq, struct mddev *mddev)
1902	{
1903	struct r10conf *conf = mddev->private;
1904	int i;
1905
1906	lockdep_assert_held(&mddev->lock);
1907
1908	if (conf->geo.near_copies < conf->geo.raid_disks)
1909	seq_printf(m: seq, fmt: " %dK chunks", mddev->chunk_sectors / 2);
1910	if (conf->geo.near_copies > 1)
1911	seq_printf(m: seq, fmt: " %d near-copies", conf->geo.near_copies);
1912	if (conf->geo.far_copies > 1) {
1913	if (conf->geo.far_offset)
1914	seq_printf(m: seq, fmt: " %d offset-copies", conf->geo.far_copies);
1915	else
1916	seq_printf(m: seq, fmt: " %d far-copies", conf->geo.far_copies);
1917	if (conf->geo.far_set_size != conf->geo.raid_disks)
1918	seq_printf(m: seq, fmt: " %d devices per set", conf->geo.far_set_size);
1919	}
1920	seq_printf(m: seq, fmt: " [%d/%d] [", conf->geo.raid_disks,
1921	conf->geo.raid_disks - mddev->degraded);
1922	for (i = 0; i < conf->geo.raid_disks; i++) {
1923	struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev);
1924
1925	seq_printf(m: seq, fmt: "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1926	}
1927	seq_printf(m: seq, fmt: "]");
1928	}
1929
1930	/* check if there are enough drives for
1931	* every block to appear on atleast one.
1932	* Don't consider the device numbered 'ignore'
1933	* as we might be about to remove it.
1934	*/
1935	static int _enough(struct r10conf *conf, int previous, int ignore)
1936	{
1937	int first = 0;
1938	int has_enough = 0;
1939	int disks, ncopies;
1940	if (previous) {
1941	disks = conf->prev.raid_disks;
1942	ncopies = conf->prev.near_copies;
1943	} else {
1944	disks = conf->geo.raid_disks;
1945	ncopies = conf->geo.near_copies;
1946	}
1947
1948	do {
1949	int n = conf->copies;
1950	int cnt = 0;
1951	int this = first;
1952	while (n--) {
1953	struct md_rdev *rdev;
1954	if (this != ignore &&
1955	(rdev = conf->mirrors[this].rdev) &&
1956	test_bit(In_sync, &rdev->flags))
1957	cnt++;
1958	this = (this+1) % disks;
1959	}
1960	if (cnt == 0)
1961	goto out;
1962	first = (first + ncopies) % disks;
1963	} while (first != 0);
1964	has_enough = 1;
1965	out:
1966	return has_enough;
1967	}
1968
1969	static int enough(struct r10conf *conf, int ignore)
1970	{
1971	/* when calling 'enough', both 'prev' and 'geo' must
1972	* be stable.
1973	* This is ensured if ->reconfig_mutex or ->device_lock
1974	* is held.
1975	*/
1976	return _enough(conf, previous: 0, ignore) &&
1977	_enough(conf, previous: 1, ignore);
1978	}
1979
1980	/**
1981	* raid10_error() - RAID10 error handler.
1982	* @mddev: affected md device.
1983	* @rdev: member device to fail.
1984	*
1985	* The routine acknowledges &rdev failure and determines new @mddev state.
1986	* If it failed, then:
1987	* - &MD_BROKEN flag is set in &mddev->flags.
1988	* Otherwise, it must be degraded:
1989	* - recovery is interrupted.
1990	* - &mddev->degraded is bumped.
1991	*
1992	* @rdev is marked as &Faulty excluding case when array is failed and
1993	* &mddev->fail_last_dev is off.
1994	*/
1995	static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
1996	{
1997	struct r10conf *conf = mddev->private;
1998	unsigned long flags;
1999
2000	spin_lock_irqsave(&conf->device_lock, flags);
2001
2002	if (test_bit(In_sync, &rdev->flags) && !enough(conf, ignore: rdev->raid_disk)) {
2003	set_bit(nr: MD_BROKEN, addr: &mddev->flags);
2004
2005	if (!mddev->fail_last_dev) {
2006	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2007	return;
2008	}
2009	}
2010	if (test_and_clear_bit(nr: In_sync, addr: &rdev->flags))
2011	mddev->degraded++;
2012
2013	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
2014	set_bit(nr: Blocked, addr: &rdev->flags);
2015	set_bit(nr: Faulty, addr: &rdev->flags);
2016	set_mask_bits(&mddev->sb_flags, 0,
2017	BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
2018	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2019	pr_crit("md/raid10:%s: Disk failure on %pg, disabling device.\n"
2020	"md/raid10:%s: Operation continuing on %d devices.\n",
2021	mdname(mddev), rdev->bdev,
2022	mdname(mddev), conf->geo.raid_disks - mddev->degraded);
2023	}
2024
2025	static void print_conf(struct r10conf *conf)
2026	{
2027	int i;
2028	struct md_rdev *rdev;
2029
2030	pr_debug("RAID10 conf printout:\n");
2031	if (!conf) {
2032	pr_debug("(!conf)\n");
2033	return;
2034	}
2035	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
2036	conf->geo.raid_disks);
2037
2038	lockdep_assert_held(&conf->mddev->reconfig_mutex);
2039	for (i = 0; i < conf->geo.raid_disks; i++) {
2040	rdev = conf->mirrors[i].rdev;
2041	if (rdev)
2042	pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n",
2043	i, !test_bit(In_sync, &rdev->flags),
2044	!test_bit(Faulty, &rdev->flags),
2045	rdev->bdev);
2046	}
2047	}
2048
2049	static void close_sync(struct r10conf *conf)
2050	{
2051	wait_barrier(conf, nowait: false);
2052	allow_barrier(conf);
2053
2054	mempool_exit(pool: &conf->r10buf_pool);
2055	}
2056
2057	static int raid10_spare_active(struct mddev *mddev)
2058	{
2059	int i;
2060	struct r10conf *conf = mddev->private;
2061	struct raid10_info *tmp;
2062	int count = 0;
2063	unsigned long flags;
2064
2065	/*
2066	* Find all non-in_sync disks within the RAID10 configuration
2067	* and mark them in_sync
2068	*/
2069	for (i = 0; i < conf->geo.raid_disks; i++) {
2070	tmp = conf->mirrors + i;
2071	if (tmp->replacement
2072	&& tmp->replacement->recovery_offset == MaxSector
2073	&& !test_bit(Faulty, &tmp->replacement->flags)
2074	&& !test_and_set_bit(nr: In_sync, addr: &tmp->replacement->flags)) {
2075	/* Replacement has just become active */
2076	if (!tmp->rdev
2077	|| !test_and_clear_bit(nr: In_sync, addr: &tmp->rdev->flags))
2078	count++;
2079	if (tmp->rdev) {
2080	/* Replaced device not technically faulty,
2081	* but we need to be sure it gets removed
2082	* and never re-added.
2083	*/
2084	set_bit(nr: Faulty, addr: &tmp->rdev->flags);
2085	sysfs_notify_dirent_safe(
2086	sd: tmp->rdev->sysfs_state);
2087	}
2088	sysfs_notify_dirent_safe(sd: tmp->replacement->sysfs_state);
2089	} else if (tmp->rdev
2090	&& tmp->rdev->recovery_offset == MaxSector
2091	&& !test_bit(Faulty, &tmp->rdev->flags)
2092	&& !test_and_set_bit(nr: In_sync, addr: &tmp->rdev->flags)) {
2093	count++;
2094	sysfs_notify_dirent_safe(sd: tmp->rdev->sysfs_state);
2095	}
2096	}
2097	spin_lock_irqsave(&conf->device_lock, flags);
2098	mddev->degraded -= count;
2099	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2100
2101	print_conf(conf);
2102	return count;
2103	}
2104
2105	static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
2106	{
2107	struct r10conf *conf = mddev->private;
2108	int err = -EEXIST;
2109	int mirror, repl_slot = -1;
2110	int first = 0;
2111	int last = conf->geo.raid_disks - 1;
2112	struct raid10_info *p;
2113
2114	if (mddev->resync_offset < MaxSector)
2115	/* only hot-add to in-sync arrays, as recovery is
2116	* very different from resync
2117	*/
2118	return -EBUSY;
2119	if (rdev->saved_raid_disk < 0 && !_enough(conf, previous: 1, ignore: -1))
2120	return -EINVAL;
2121
2122	if (rdev->raid_disk >= 0)
2123	first = last = rdev->raid_disk;
2124
2125	if (rdev->saved_raid_disk >= first &&
2126	rdev->saved_raid_disk < conf->geo.raid_disks &&
2127	conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
2128	mirror = rdev->saved_raid_disk;
2129	else
2130	mirror = first;
2131	for ( ; mirror <= last ; mirror++) {
2132	p = &conf->mirrors[mirror];
2133	if (p->recovery_disabled == mddev->recovery_disabled)
2134	continue;
2135	if (p->rdev) {
2136	if (test_bit(WantReplacement, &p->rdev->flags) &&
2137	p->replacement == NULL && repl_slot < 0)
2138	repl_slot = mirror;
2139	continue;
2140	}
2141
2142	err = mddev_stack_new_rdev(mddev, rdev);
2143	if (err)
2144	return err;
2145	p->head_position = 0;
2146	p->recovery_disabled = mddev->recovery_disabled - 1;
2147	rdev->raid_disk = mirror;
2148	err = 0;
2149	if (rdev->saved_raid_disk != mirror)
2150	conf->fullsync = 1;
2151	WRITE_ONCE(p->rdev, rdev);
2152	break;
2153	}
2154
2155	if (err && repl_slot >= 0) {
2156	p = &conf->mirrors[repl_slot];
2157	clear_bit(nr: In_sync, addr: &rdev->flags);
2158	set_bit(nr: Replacement, addr: &rdev->flags);
2159	rdev->raid_disk = repl_slot;
2160	err = mddev_stack_new_rdev(mddev, rdev);
2161	if (err)
2162	return err;
2163	conf->fullsync = 1;
2164	WRITE_ONCE(p->replacement, rdev);
2165	}
2166
2167	print_conf(conf);
2168	return err;
2169	}
2170
2171	static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
2172	{
2173	struct r10conf *conf = mddev->private;
2174	int err = 0;
2175	int number = rdev->raid_disk;
2176	struct md_rdev **rdevp;
2177	struct raid10_info *p;
2178
2179	print_conf(conf);
2180	if (unlikely(number >= mddev->raid_disks))
2181	return 0;
2182	p = conf->mirrors + number;
2183	if (rdev == p->rdev)
2184	rdevp = &p->rdev;
2185	else if (rdev == p->replacement)
2186	rdevp = &p->replacement;
2187	else
2188	return 0;
2189
2190	if (test_bit(In_sync, &rdev->flags) ||
2191	atomic_read(v: &rdev->nr_pending)) {
2192	err = -EBUSY;
2193	goto abort;
2194	}
2195	/* Only remove non-faulty devices if recovery
2196	* is not possible.
2197	*/
2198	if (!test_bit(Faulty, &rdev->flags) &&
2199	mddev->recovery_disabled != p->recovery_disabled &&
2200	(!p->replacement || p->replacement == rdev) &&
2201	number < conf->geo.raid_disks &&
2202	enough(conf, ignore: -1)) {
2203	err = -EBUSY;
2204	goto abort;
2205	}
2206	WRITE_ONCE(*rdevp, NULL);
2207	if (p->replacement) {
2208	/* We must have just cleared 'rdev' */
2209	WRITE_ONCE(p->rdev, p->replacement);
2210	clear_bit(nr: Replacement, addr: &p->replacement->flags);
2211	WRITE_ONCE(p->replacement, NULL);
2212	}
2213
2214	clear_bit(nr: WantReplacement, addr: &rdev->flags);
2215	err = md_integrity_register(mddev);
2216
2217	abort:
2218
2219	print_conf(conf);
2220	return err;
2221	}
2222
2223	static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)
2224	{
2225	struct r10conf *conf = r10_bio->mddev->private;
2226
2227	if (!bio->bi_status)
2228	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
2229	else
2230	/* The write handler will notice the lack of
2231	* R10BIO_Uptodate and record any errors etc
2232	*/
2233	atomic_add(i: r10_bio->sectors,
2234	v: &conf->mirrors[d].rdev->corrected_errors);
2235
2236	/* for reconstruct, we always reschedule after a read.
2237	* for resync, only after all reads
2238	*/
2239	rdev_dec_pending(rdev: conf->mirrors[d].rdev, mddev: conf->mddev);
2240	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
2241	atomic_dec_and_test(v: &r10_bio->remaining)) {
2242	/* we have read all the blocks,
2243	* do the comparison in process context in raid10d
2244	*/
2245	reschedule_retry(r10_bio);
2246	}
2247	}
2248
2249	static void end_sync_read(struct bio *bio)
2250	{
2251	struct r10bio *r10_bio = get_resync_r10bio(bio);
2252	struct r10conf *conf = r10_bio->mddev->private;
2253	int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
2254
2255	__end_sync_read(r10_bio, bio, d);
2256	}
2257
2258	static void end_reshape_read(struct bio *bio)
2259	{
2260	/* reshape read bio isn't allocated from r10buf_pool */
2261	struct r10bio *r10_bio = bio->bi_private;
2262
2263	__end_sync_read(r10_bio, bio, d: r10_bio->read_slot);
2264	}
2265
2266	static void end_sync_request(struct r10bio *r10_bio)
2267	{
2268	struct mddev *mddev = r10_bio->mddev;
2269
2270	while (atomic_dec_and_test(v: &r10_bio->remaining)) {
2271	if (r10_bio->master_bio == NULL) {
2272	/* the primary of several recovery bios */
2273	sector_t s = r10_bio->sectors;
2274	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2275	test_bit(R10BIO_WriteError, &r10_bio->state))
2276	reschedule_retry(r10_bio);
2277	else
2278	put_buf(r10_bio);
2279	md_done_sync(mddev, blocks: s, ok: 1);
2280	break;
2281	} else {
2282	struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
2283	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2284	test_bit(R10BIO_WriteError, &r10_bio->state))
2285	reschedule_retry(r10_bio);
2286	else
2287	put_buf(r10_bio);
2288	r10_bio = r10_bio2;
2289	}
2290	}
2291	}
2292
2293	static void end_sync_write(struct bio *bio)
2294	{
2295	struct r10bio *r10_bio = get_resync_r10bio(bio);
2296	struct mddev *mddev = r10_bio->mddev;
2297	struct r10conf *conf = mddev->private;
2298	int d;
2299	int slot;
2300	int repl;
2301	struct md_rdev *rdev = NULL;
2302
2303	d = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
2304	if (repl)
2305	rdev = conf->mirrors[d].replacement;
2306	else
2307	rdev = conf->mirrors[d].rdev;
2308
2309	if (bio->bi_status) {
2310	if (repl)
2311	md_error(mddev, rdev);
2312	else {
2313	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2314	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
2315	set_bit(nr: MD_RECOVERY_NEEDED,
2316	addr: &rdev->mddev->recovery);
2317	set_bit(nr: R10BIO_WriteError, addr: &r10_bio->state);
2318	}
2319	} else if (rdev_has_badblock(rdev, s: r10_bio->devs[slot].addr,
2320	sectors: r10_bio->sectors)) {
2321	set_bit(nr: R10BIO_MadeGood, addr: &r10_bio->state);
2322	}
2323
2324	rdev_dec_pending(rdev, mddev);
2325
2326	end_sync_request(r10_bio);
2327	}
2328
2329	/*
2330	* Note: sync and recover and handled very differently for raid10
2331	* This code is for resync.
2332	* For resync, we read through virtual addresses and read all blocks.
2333	* If there is any error, we schedule a write. The lowest numbered
2334	* drive is authoritative.
2335	* However requests come for physical address, so we need to map.
2336	* For every physical address there are raid_disks/copies virtual addresses,
2337	* which is always are least one, but is not necessarly an integer.
2338	* This means that a physical address can span multiple chunks, so we may
2339	* have to submit multiple io requests for a single sync request.
2340	*/
2341	/*
2342	* We check if all blocks are in-sync and only write to blocks that
2343	* aren't in sync
2344	*/
2345	static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2346	{
2347	struct r10conf *conf = mddev->private;
2348	int i, first;
2349	struct bio *tbio, *fbio;
2350	int vcnt;
2351	struct page **tpages, **fpages;
2352
2353	atomic_set(v: &r10_bio->remaining, i: 1);
2354
2355	/* find the first device with a block */
2356	for (i=0; i<conf->copies; i++)
2357	if (!r10_bio->devs[i].bio->bi_status)
2358	break;
2359
2360	if (i == conf->copies)
2361	goto done;
2362
2363	first = i;
2364	fbio = r10_bio->devs[i].bio;
2365	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
2366	fbio->bi_iter.bi_idx = 0;
2367	fpages = get_resync_pages(bio: fbio)->pages;
2368
2369	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
2370	/* now find blocks with errors */
2371	for (i=0 ; i < conf->copies ; i++) {
2372	int j, d;
2373	struct md_rdev *rdev;
2374	struct resync_pages *rp;
2375
2376	tbio = r10_bio->devs[i].bio;
2377
2378	if (tbio->bi_end_io != end_sync_read)
2379	continue;
2380	if (i == first)
2381	continue;
2382
2383	tpages = get_resync_pages(bio: tbio)->pages;
2384	d = r10_bio->devs[i].devnum;
2385	rdev = conf->mirrors[d].rdev;
2386	if (!r10_bio->devs[i].bio->bi_status) {
2387	/* We know that the bi_io_vec layout is the same for
2388	* both 'first' and 'i', so we just compare them.
2389	* All vec entries are PAGE_SIZE;
2390	*/
2391	int sectors = r10_bio->sectors;
2392	for (j = 0; j < vcnt; j++) {
2393	int len = PAGE_SIZE;
2394	if (sectors < (len / 512))
2395	len = sectors * 512;
2396	if (memcmp(page_address(fpages[j]),
2397	page_address(tpages[j]),
2398	size: len))
2399	break;
2400	sectors -= len/512;
2401	}
2402	if (j == vcnt)
2403	continue;
2404	atomic64_add(i: r10_bio->sectors, v: &mddev->resync_mismatches);
2405	if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2406	/* Don't fix anything. */
2407	continue;
2408	} else if (test_bit(FailFast, &rdev->flags)) {
2409	/* Just give up on this device */
2410	md_error(mddev: rdev->mddev, rdev);
2411	continue;
2412	}
2413	/* Ok, we need to write this bio, either to correct an
2414	* inconsistency or to correct an unreadable block.
2415	* First we need to fixup bv_offset, bv_len and
2416	* bi_vecs, as the read request might have corrupted these
2417	*/
2418	rp = get_resync_pages(bio: tbio);
2419	bio_reset(bio: tbio, bdev: conf->mirrors[d].rdev->bdev, opf: REQ_OP_WRITE);
2420
2421	md_bio_reset_resync_pages(bio: tbio, rp, size: fbio->bi_iter.bi_size);
2422
2423	rp->raid_bio = r10_bio;
2424	tbio->bi_private = rp;
2425	tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
2426	tbio->bi_end_io = end_sync_write;
2427
2428	bio_copy_data(dst: tbio, src: fbio);
2429
2430	atomic_inc(v: &conf->mirrors[d].rdev->nr_pending);
2431	atomic_inc(v: &r10_bio->remaining);
2432
2433	if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
2434	tbio->bi_opf |= MD_FAILFAST;
2435	tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
2436	submit_bio_noacct(bio: tbio);
2437	}
2438
2439	/* Now write out to any replacement devices
2440	* that are active
2441	*/
2442	for (i = 0; i < conf->copies; i++) {
2443	tbio = r10_bio->devs[i].repl_bio;
2444	if (!tbio || !tbio->bi_end_io)
2445	continue;
2446	if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2447	&& r10_bio->devs[i].bio != fbio)
2448	bio_copy_data(dst: tbio, src: fbio);
2449	atomic_inc(v: &r10_bio->remaining);
2450	submit_bio_noacct(bio: tbio);
2451	}
2452
2453	done:
2454	if (atomic_dec_and_test(v: &r10_bio->remaining)) {
2455	md_done_sync(mddev, blocks: r10_bio->sectors, ok: 1);
2456	put_buf(r10_bio);
2457	}
2458	}
2459
2460	/*
2461	* Now for the recovery code.
2462	* Recovery happens across physical sectors.
2463	* We recover all non-is_sync drives by finding the virtual address of
2464	* each, and then choose a working drive that also has that virt address.
2465	* There is a separate r10_bio for each non-in_sync drive.
2466	* Only the first two slots are in use. The first for reading,
2467	* The second for writing.
2468	*
2469	*/
2470	static void fix_recovery_read_error(struct r10bio *r10_bio)
2471	{
2472	/* We got a read error during recovery.
2473	* We repeat the read in smaller page-sized sections.
2474	* If a read succeeds, write it to the new device or record
2475	* a bad block if we cannot.
2476	* If a read fails, record a bad block on both old and
2477	* new devices.
2478	*/
2479	struct mddev *mddev = r10_bio->mddev;
2480	struct r10conf *conf = mddev->private;
2481	struct bio *bio = r10_bio->devs[0].bio;
2482	sector_t sect = 0;
2483	int sectors = r10_bio->sectors;
2484	int idx = 0;
2485	int dr = r10_bio->devs[0].devnum;
2486	int dw = r10_bio->devs[1].devnum;
2487	struct page **pages = get_resync_pages(bio)->pages;
2488
2489	while (sectors) {
2490	int s = sectors;
2491	struct md_rdev *rdev;
2492	sector_t addr;
2493	int ok;
2494
2495	if (s > (PAGE_SIZE>>9))
2496	s = PAGE_SIZE >> 9;
2497
2498	rdev = conf->mirrors[dr].rdev;
2499	addr = r10_bio->devs[0].addr + sect;
2500	ok = sync_page_io(rdev,
2501	sector: addr,
2502	size: s << 9,
2503	page: pages[idx],
2504	opf: REQ_OP_READ, metadata_op: false);
2505	if (ok) {
2506	rdev = conf->mirrors[dw].rdev;
2507	addr = r10_bio->devs[1].addr + sect;
2508	ok = sync_page_io(rdev,
2509	sector: addr,
2510	size: s << 9,
2511	page: pages[idx],
2512	opf: REQ_OP_WRITE, metadata_op: false);
2513	if (!ok) {
2514	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2515	if (!test_and_set_bit(nr: WantReplacement,
2516	addr: &rdev->flags))
2517	set_bit(nr: MD_RECOVERY_NEEDED,
2518	addr: &rdev->mddev->recovery);
2519	}
2520	}
2521	if (!ok) {
2522	/* We don't worry if we cannot set a bad block -
2523	* it really is bad so there is no loss in not
2524	* recording it yet
2525	*/
2526	rdev_set_badblocks(rdev, s: addr, sectors: s, is_new: 0);
2527
2528	if (rdev != conf->mirrors[dw].rdev) {
2529	/* need bad block on destination too */
2530	struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2531	addr = r10_bio->devs[1].addr + sect;
2532	ok = rdev_set_badblocks(rdev: rdev2, s: addr, sectors: s, is_new: 0);
2533	if (!ok) {
2534	/* just abort the recovery */
2535	pr_notice("md/raid10:%s: recovery aborted due to read error\n",
2536	mdname(mddev));
2537
2538	conf->mirrors[dw].recovery_disabled
2539	= mddev->recovery_disabled;
2540	set_bit(nr: MD_RECOVERY_INTR,
2541	addr: &mddev->recovery);
2542	break;
2543	}
2544	}
2545	}
2546
2547	sectors -= s;
2548	sect += s;
2549	idx++;
2550	}
2551	}
2552
2553	static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2554	{
2555	struct r10conf *conf = mddev->private;
2556	int d;
2557	struct bio *wbio = r10_bio->devs[1].bio;
2558	struct bio *wbio2 = r10_bio->devs[1].repl_bio;
2559
2560	/* Need to test wbio2->bi_end_io before we call
2561	* submit_bio_noacct as if the former is NULL,
2562	* the latter is free to free wbio2.
2563	*/
2564	if (wbio2 && !wbio2->bi_end_io)
2565	wbio2 = NULL;
2566
2567	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2568	fix_recovery_read_error(r10_bio);
2569	if (wbio->bi_end_io)
2570	end_sync_request(r10_bio);
2571	if (wbio2)
2572	end_sync_request(r10_bio);
2573	return;
2574	}
2575
2576	/*
2577	* share the pages with the first bio
2578	* and submit the write request
2579	*/
2580	d = r10_bio->devs[1].devnum;
2581	if (wbio->bi_end_io) {
2582	atomic_inc(v: &conf->mirrors[d].rdev->nr_pending);
2583	submit_bio_noacct(bio: wbio);
2584	}
2585	if (wbio2) {
2586	atomic_inc(v: &conf->mirrors[d].replacement->nr_pending);
2587	submit_bio_noacct(bio: wbio2);
2588	}
2589	}
2590
2591	static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2592	int sectors, struct page *page, enum req_op op)
2593	{
2594	if (rdev_has_badblock(rdev, s: sector, sectors) &&
2595	(op == REQ_OP_READ || test_bit(WriteErrorSeen, &rdev->flags)))
2596	return -1;
2597	if (sync_page_io(rdev, sector, size: sectors << 9, page, opf: op, metadata_op: false))
2598	/* success */
2599	return 1;
2600	if (op == REQ_OP_WRITE) {
2601	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2602	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
2603	set_bit(nr: MD_RECOVERY_NEEDED,
2604	addr: &rdev->mddev->recovery);
2605	}
2606	/* need to record an error - either for the block or the device */
2607	if (!rdev_set_badblocks(rdev, s: sector, sectors, is_new: 0))
2608	md_error(mddev: rdev->mddev, rdev);
2609	return 0;
2610	}
2611
2612	/*
2613	* This is a kernel thread which:
2614	*
2615	* 1. Retries failed read operations on working mirrors.
2616	* 2. Updates the raid superblock when problems encounter.
2617	* 3. Performs writes following reads for array synchronising.
2618	*/
2619
2620	static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2621	{
2622	int sect = 0; /* Offset from r10_bio->sector */
2623	int sectors = r10_bio->sectors, slot = r10_bio->read_slot;
2624	struct md_rdev *rdev;
2625	int d = r10_bio->devs[slot].devnum;
2626
2627	/* still own a reference to this rdev, so it cannot
2628	* have been cleared recently.
2629	*/
2630	rdev = conf->mirrors[d].rdev;
2631
2632	if (test_bit(Faulty, &rdev->flags))
2633	/* drive has already been failed, just ignore any
2634	more fix_read_error() attempts */
2635	return;
2636
2637	if (exceed_read_errors(mddev, rdev)) {
2638	r10_bio->devs[slot].bio = IO_BLOCKED;
2639	return;
2640	}
2641
2642	while(sectors) {
2643	int s = sectors;
2644	int sl = slot;
2645	int success = 0;
2646	int start;
2647
2648	if (s > (PAGE_SIZE>>9))
2649	s = PAGE_SIZE >> 9;
2650
2651	do {
2652	d = r10_bio->devs[sl].devnum;
2653	rdev = conf->mirrors[d].rdev;
2654	if (rdev &&
2655	test_bit(In_sync, &rdev->flags) &&
2656	!test_bit(Faulty, &rdev->flags) &&
2657	rdev_has_badblock(rdev,
2658	s: r10_bio->devs[sl].addr + sect,
2659	sectors: s) == 0) {
2660	atomic_inc(v: &rdev->nr_pending);
2661	success = sync_page_io(rdev,
2662	sector: r10_bio->devs[sl].addr +
2663	sect,
2664	size: s<<9,
2665	page: conf->tmppage,
2666	opf: REQ_OP_READ, metadata_op: false);
2667	rdev_dec_pending(rdev, mddev);
2668	if (success)
2669	break;
2670	}
2671	sl++;
2672	if (sl == conf->copies)
2673	sl = 0;
2674	} while (sl != slot);
2675
2676	if (!success) {
2677	/* Cannot read from anywhere, just mark the block
2678	* as bad on the first device to discourage future
2679	* reads.
2680	*/
2681	int dn = r10_bio->devs[slot].devnum;
2682	rdev = conf->mirrors[dn].rdev;
2683
2684	if (!rdev_set_badblocks(
2685	rdev,
2686	s: r10_bio->devs[slot].addr
2687	+ sect,
2688	sectors: s, is_new: 0)) {
2689	md_error(mddev, rdev);
2690	r10_bio->devs[slot].bio
2691	= IO_BLOCKED;
2692	}
2693	break;
2694	}
2695
2696	start = sl;
2697	/* write it back and re-read */
2698	while (sl != slot) {
2699	if (sl==0)
2700	sl = conf->copies;
2701	sl--;
2702	d = r10_bio->devs[sl].devnum;
2703	rdev = conf->mirrors[d].rdev;
2704	if (!rdev ||
2705	test_bit(Faulty, &rdev->flags) ||
2706	!test_bit(In_sync, &rdev->flags))
2707	continue;
2708
2709	atomic_inc(v: &rdev->nr_pending);
2710	if (r10_sync_page_io(rdev,
2711	sector: r10_bio->devs[sl].addr +
2712	sect,
2713	sectors: s, page: conf->tmppage, op: REQ_OP_WRITE)
2714	== 0) {
2715	/* Well, this device is dead */
2716	pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %pg)\n",
2717	mdname(mddev), s,
2718	(unsigned long long)(
2719	sect +
2720	choose_data_offset(r10_bio,
2721	rdev)),
2722	rdev->bdev);
2723	pr_notice("md/raid10:%s: %pg: failing drive\n",
2724	mdname(mddev),
2725	rdev->bdev);
2726	}
2727	rdev_dec_pending(rdev, mddev);
2728	}
2729	sl = start;
2730	while (sl != slot) {
2731	if (sl==0)
2732	sl = conf->copies;
2733	sl--;
2734	d = r10_bio->devs[sl].devnum;
2735	rdev = conf->mirrors[d].rdev;
2736	if (!rdev ||
2737	test_bit(Faulty, &rdev->flags) ||
2738	!test_bit(In_sync, &rdev->flags))
2739	continue;
2740
2741	atomic_inc(v: &rdev->nr_pending);
2742	switch (r10_sync_page_io(rdev,
2743	sector: r10_bio->devs[sl].addr +
2744	sect,
2745	sectors: s, page: conf->tmppage, op: REQ_OP_READ)) {
2746	case 0:
2747	/* Well, this device is dead */
2748	pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %pg)\n",
2749	mdname(mddev), s,
2750	(unsigned long long)(
2751	sect +
2752	choose_data_offset(r10_bio, rdev)),
2753	rdev->bdev);
2754	pr_notice("md/raid10:%s: %pg: failing drive\n",
2755	mdname(mddev),
2756	rdev->bdev);
2757	break;
2758	case 1:
2759	pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %pg)\n",
2760	mdname(mddev), s,
2761	(unsigned long long)(
2762	sect +
2763	choose_data_offset(r10_bio, rdev)),
2764	rdev->bdev);
2765	atomic_add(i: s, v: &rdev->corrected_errors);
2766	}
2767
2768	rdev_dec_pending(rdev, mddev);
2769	}
2770
2771	sectors -= s;
2772	sect += s;
2773	}
2774	}
2775
2776	static bool narrow_write_error(struct r10bio *r10_bio, int i)
2777	{
2778	struct bio *bio = r10_bio->master_bio;
2779	struct mddev *mddev = r10_bio->mddev;
2780	struct r10conf *conf = mddev->private;
2781	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2782	/* bio has the data to be written to slot 'i' where
2783	* we just recently had a write error.
2784	* We repeatedly clone the bio and trim down to one block,
2785	* then try the write. Where the write fails we record
2786	* a bad block.
2787	* It is conceivable that the bio doesn't exactly align with
2788	* blocks. We must handle this.
2789	*
2790	* We currently own a reference to the rdev.
2791	*/
2792
2793	int block_sectors;
2794	sector_t sector;
2795	int sectors;
2796	int sect_to_write = r10_bio->sectors;
2797	bool ok = true;
2798
2799	if (rdev->badblocks.shift < 0)
2800	return false;
2801
2802	block_sectors = roundup(1 << rdev->badblocks.shift,
2803	bdev_logical_block_size(rdev->bdev) >> 9);
2804	sector = r10_bio->sector;
2805	sectors = ((r10_bio->sector + block_sectors)
2806	& ~(sector_t)(block_sectors - 1))
2807	- sector;
2808
2809	while (sect_to_write) {
2810	struct bio *wbio;
2811	sector_t wsector;
2812	if (sectors > sect_to_write)
2813	sectors = sect_to_write;
2814	/* Write at 'sector' for 'sectors' */
2815	wbio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, GFP_NOIO,
2816	bs: &mddev->bio_set);
2817	bio_trim(bio: wbio, offset: sector - bio->bi_iter.bi_sector, size: sectors);
2818	wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
2819	wbio->bi_iter.bi_sector = wsector +
2820	choose_data_offset(r10_bio, rdev);
2821	wbio->bi_opf = REQ_OP_WRITE;
2822
2823	if (submit_bio_wait(bio: wbio) < 0)
2824	/* Failure! */
2825	ok = rdev_set_badblocks(rdev, s: wsector,
2826	sectors, is_new: 0)
2827	&& ok;
2828
2829	bio_put(wbio);
2830	sect_to_write -= sectors;
2831	sector += sectors;
2832	sectors = block_sectors;
2833	}
2834	return ok;
2835	}
2836
2837	static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2838	{
2839	int slot = r10_bio->read_slot;
2840	struct bio *bio;
2841	struct r10conf *conf = mddev->private;
2842	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2843
2844	/* we got a read error. Maybe the drive is bad. Maybe just
2845	* the block and we can fix it.
2846	* We freeze all other IO, and try reading the block from
2847	* other devices. When we find one, we re-write
2848	* and check it that fixes the read error.
2849	* This is all done synchronously while the array is
2850	* frozen.
2851	*/
2852	bio = r10_bio->devs[slot].bio;
2853	bio_put(bio);
2854	r10_bio->devs[slot].bio = NULL;
2855
2856	if (mddev->ro)
2857	r10_bio->devs[slot].bio = IO_BLOCKED;
2858	else if (!test_bit(FailFast, &rdev->flags)) {
2859	freeze_array(conf, extra: 1);
2860	fix_read_error(conf, mddev, r10_bio);
2861	unfreeze_array(conf);
2862	} else
2863	md_error(mddev, rdev);
2864
2865	rdev_dec_pending(rdev, mddev);
2866	r10_bio->state = 0;
2867	raid10_read_request(mddev, bio: r10_bio->master_bio, r10_bio, io_accounting: false);
2868	/*
2869	* allow_barrier after re-submit to ensure no sync io
2870	* can be issued while regular io pending.
2871	*/
2872	allow_barrier(conf);
2873	}
2874
2875	static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2876	{
2877	/* Some sort of write request has finished and it
2878	* succeeded in writing where we thought there was a
2879	* bad block. So forget the bad block.
2880	* Or possibly if failed and we need to record
2881	* a bad block.
2882	*/
2883	int m;
2884	struct md_rdev *rdev;
2885
2886	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2887	test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2888	for (m = 0; m < conf->copies; m++) {
2889	int dev = r10_bio->devs[m].devnum;
2890	rdev = conf->mirrors[dev].rdev;
2891	if (r10_bio->devs[m].bio == NULL ||
2892	r10_bio->devs[m].bio->bi_end_io == NULL)
2893	continue;
2894	if (!r10_bio->devs[m].bio->bi_status) {
2895	rdev_clear_badblocks(
2896	rdev,
2897	s: r10_bio->devs[m].addr,
2898	sectors: r10_bio->sectors, is_new: 0);
2899	} else {
2900	if (!rdev_set_badblocks(
2901	rdev,
2902	s: r10_bio->devs[m].addr,
2903	sectors: r10_bio->sectors, is_new: 0))
2904	md_error(mddev: conf->mddev, rdev);
2905	}
2906	rdev = conf->mirrors[dev].replacement;
2907	if (r10_bio->devs[m].repl_bio == NULL ||
2908	r10_bio->devs[m].repl_bio->bi_end_io == NULL)
2909	continue;
2910
2911	if (!r10_bio->devs[m].repl_bio->bi_status) {
2912	rdev_clear_badblocks(
2913	rdev,
2914	s: r10_bio->devs[m].addr,
2915	sectors: r10_bio->sectors, is_new: 0);
2916	} else {
2917	if (!rdev_set_badblocks(
2918	rdev,
2919	s: r10_bio->devs[m].addr,
2920	sectors: r10_bio->sectors, is_new: 0))
2921	md_error(mddev: conf->mddev, rdev);
2922	}
2923	}
2924	put_buf(r10_bio);
2925	} else {
2926	bool fail = false;
2927	for (m = 0; m < conf->copies; m++) {
2928	int dev = r10_bio->devs[m].devnum;
2929	struct bio *bio = r10_bio->devs[m].bio;
2930	rdev = conf->mirrors[dev].rdev;
2931	if (bio == IO_MADE_GOOD) {
2932	rdev_clear_badblocks(
2933	rdev,
2934	s: r10_bio->devs[m].addr,
2935	sectors: r10_bio->sectors, is_new: 0);
2936	rdev_dec_pending(rdev, mddev: conf->mddev);
2937	} else if (bio != NULL && bio->bi_status) {
2938	fail = true;
2939	if (!narrow_write_error(r10_bio, i: m))
2940	md_error(mddev: conf->mddev, rdev);
2941	rdev_dec_pending(rdev, mddev: conf->mddev);
2942	}
2943	bio = r10_bio->devs[m].repl_bio;
2944	rdev = conf->mirrors[dev].replacement;
2945	if (rdev && bio == IO_MADE_GOOD) {
2946	rdev_clear_badblocks(
2947	rdev,
2948	s: r10_bio->devs[m].addr,
2949	sectors: r10_bio->sectors, is_new: 0);
2950	rdev_dec_pending(rdev, mddev: conf->mddev);
2951	}
2952	}
2953	if (fail) {
2954	spin_lock_irq(lock: &conf->device_lock);
2955	list_add(new: &r10_bio->retry_list, head: &conf->bio_end_io_list);
2956	conf->nr_queued++;
2957	spin_unlock_irq(lock: &conf->device_lock);
2958	/*
2959	* In case freeze_array() is waiting for condition
2960	* nr_pending == nr_queued + extra to be true.
2961	*/
2962	wake_up(&conf->wait_barrier);
2963	md_wakeup_thread(conf->mddev->thread);
2964	} else {
2965	if (test_bit(R10BIO_WriteError,
2966	&r10_bio->state))
2967	close_write(r10_bio);
2968	raid_end_bio_io(r10_bio);
2969	}
2970	}
2971	}
2972
2973	static void raid10d(struct md_thread *thread)
2974	{
2975	struct mddev *mddev = thread->mddev;
2976	struct r10bio *r10_bio;
2977	unsigned long flags;
2978	struct r10conf *conf = mddev->private;
2979	struct list_head *head = &conf->retry_list;
2980	struct blk_plug plug;
2981
2982	md_check_recovery(mddev);
2983
2984	if (!list_empty_careful(head: &conf->bio_end_io_list) &&
2985	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2986	LIST_HEAD(tmp);
2987	spin_lock_irqsave(&conf->device_lock, flags);
2988	if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2989	while (!list_empty(head: &conf->bio_end_io_list)) {
2990	list_move(list: conf->bio_end_io_list.prev, head: &tmp);
2991	conf->nr_queued--;
2992	}
2993	}
2994	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2995	while (!list_empty(head: &tmp)) {
2996	r10_bio = list_first_entry(&tmp, struct r10bio,
2997	retry_list);
2998	list_del(entry: &r10_bio->retry_list);
2999
3000	if (test_bit(R10BIO_WriteError,
3001	&r10_bio->state))
3002	close_write(r10_bio);
3003	raid_end_bio_io(r10_bio);
3004	}
3005	}
3006
3007	blk_start_plug(&plug);
3008	for (;;) {
3009
3010	flush_pending_writes(conf);
3011
3012	spin_lock_irqsave(&conf->device_lock, flags);
3013	if (list_empty(head)) {
3014	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
3015	break;
3016	}
3017	r10_bio = list_entry(head->prev, struct r10bio, retry_list);
3018	list_del(entry: head->prev);
3019	conf->nr_queued--;
3020	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
3021
3022	mddev = r10_bio->mddev;
3023	conf = mddev->private;
3024	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
3025	test_bit(R10BIO_WriteError, &r10_bio->state))
3026	handle_write_completed(conf, r10_bio);
3027	else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
3028	reshape_request_write(mddev, r10_bio);
3029	else if (test_bit(R10BIO_IsSync, &r10_bio->state))
3030	sync_request_write(mddev, r10_bio);
3031	else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
3032	recovery_request_write(mddev, r10_bio);
3033	else if (test_bit(R10BIO_ReadError, &r10_bio->state))
3034	handle_read_error(mddev, r10_bio);
3035	else
3036	WARN_ON_ONCE(1);
3037
3038	cond_resched();
3039	if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
3040	md_check_recovery(mddev);
3041	}
3042	blk_finish_plug(&plug);
3043	}
3044
3045	static int init_resync(struct r10conf *conf)
3046	{
3047	int ret, buffs, i;
3048
3049	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
3050	BUG_ON(mempool_initialized(&conf->r10buf_pool));
3051	conf->have_replacement = 0;
3052	for (i = 0; i < conf->geo.raid_disks; i++)
3053	if (conf->mirrors[i].replacement)
3054	conf->have_replacement = 1;
3055	ret = mempool_init(&conf->r10buf_pool, buffs,
3056	r10buf_pool_alloc, r10buf_pool_free, conf);
3057	if (ret)
3058	return ret;
3059	conf->next_resync = 0;
3060	return 0;
3061	}
3062
3063	static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)
3064	{
3065	struct r10bio *r10bio = mempool_alloc(&conf->r10buf_pool, GFP_NOIO);
3066	struct rsync_pages *rp;
3067	struct bio *bio;
3068	int nalloc;
3069	int i;
3070
3071	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
3072	test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
3073	nalloc = conf->copies; /* resync */
3074	else
3075	nalloc = 2; /* recovery */
3076
3077	for (i = 0; i < nalloc; i++) {
3078	bio = r10bio->devs[i].bio;
3079	rp = bio->bi_private;
3080	bio_reset(bio, NULL, opf: 0);
3081	bio->bi_private = rp;
3082	bio = r10bio->devs[i].repl_bio;
3083	if (bio) {
3084	rp = bio->bi_private;
3085	bio_reset(bio, NULL, opf: 0);
3086	bio->bi_private = rp;
3087	}
3088	}
3089	return r10bio;
3090	}
3091
3092	/*
3093	* Set cluster_sync_high since we need other nodes to add the
3094	* range [cluster_sync_low, cluster_sync_high] to suspend list.
3095	*/
3096	static void raid10_set_cluster_sync_high(struct r10conf *conf)
3097	{
3098	sector_t window_size;
3099	int extra_chunk, chunks;
3100
3101	/*
3102	* First, here we define "stripe" as a unit which across
3103	* all member devices one time, so we get chunks by use
3104	* raid_disks / near_copies. Otherwise, if near_copies is
3105	* close to raid_disks, then resync window could increases
3106	* linearly with the increase of raid_disks, which means
3107	* we will suspend a really large IO window while it is not
3108	* necessary. If raid_disks is not divisible by near_copies,
3109	* an extra chunk is needed to ensure the whole "stripe" is
3110	* covered.
3111	*/
3112
3113	chunks = conf->geo.raid_disks / conf->geo.near_copies;
3114	if (conf->geo.raid_disks % conf->geo.near_copies == 0)
3115	extra_chunk = 0;
3116	else
3117	extra_chunk = 1;
3118	window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors;
3119
3120	/*
3121	* At least use a 32M window to align with raid1's resync window
3122	*/
3123	window_size = (CLUSTER_RESYNC_WINDOW_SECTORS > window_size) ?
3124	CLUSTER_RESYNC_WINDOW_SECTORS : window_size;
3125
3126	conf->cluster_sync_high = conf->cluster_sync_low + window_size;
3127	}
3128
3129	/*
3130	* perform a "sync" on one "block"
3131	*
3132	* We need to make sure that no normal I/O request - particularly write
3133	* requests - conflict with active sync requests.
3134	*
3135	* This is achieved by tracking pending requests and a 'barrier' concept
3136	* that can be installed to exclude normal IO requests.
3137	*
3138	* Resync and recovery are handled very differently.
3139	* We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
3140	*
3141	* For resync, we iterate over virtual addresses, read all copies,
3142	* and update if there are differences. If only one copy is live,
3143	* skip it.
3144	* For recovery, we iterate over physical addresses, read a good
3145	* value for each non-in_sync drive, and over-write.
3146	*
3147	* So, for recovery we may have several outstanding complex requests for a
3148	* given address, one for each out-of-sync device. We model this by allocating
3149	* a number of r10_bio structures, one for each out-of-sync device.
3150	* As we setup these structures, we collect all bio's together into a list
3151	* which we then process collectively to add pages, and then process again
3152	* to pass to submit_bio_noacct.
3153	*
3154	* The r10_bio structures are linked using a borrowed master_bio pointer.
3155	* This link is counted in ->remaining. When the r10_bio that points to NULL
3156	* has its remaining count decremented to 0, the whole complex operation
3157	* is complete.
3158	*
3159	*/
3160
3161	static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
3162	sector_t max_sector, int *skipped)
3163	{
3164	struct r10conf *conf = mddev->private;
3165	struct r10bio *r10_bio;
3166	struct bio *biolist = NULL, *bio;
3167	sector_t nr_sectors;
3168	int i;
3169	int max_sync;
3170	sector_t sync_blocks;
3171	sector_t sectors_skipped = 0;
3172	int chunks_skipped = 0;
3173	sector_t chunk_mask = conf->geo.chunk_mask;
3174	int page_idx = 0;
3175	int error_disk = -1;
3176
3177	/*
3178	* Allow skipping a full rebuild for incremental assembly
3179	* of a clean array, like RAID1 does.
3180	*/
3181	if (mddev->bitmap == NULL &&
3182	mddev->resync_offset == MaxSector &&
3183	mddev->reshape_position == MaxSector &&
3184	!test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
3185	!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
3186	!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
3187	conf->fullsync == 0) {
3188	*skipped = 1;
3189	return mddev->dev_sectors - sector_nr;
3190	}
3191
3192	if (!mempool_initialized(pool: &conf->r10buf_pool))
3193	if (init_resync(conf))
3194	return 0;
3195
3196	skipped:
3197	if (sector_nr >= max_sector) {
3198	conf->cluster_sync_low = 0;
3199	conf->cluster_sync_high = 0;
3200
3201	/* If we aborted, we need to abort the
3202	* sync on the 'current' bitmap chucks (there can
3203	* be several when recovering multiple devices).
3204	* as we may have started syncing it but not finished.
3205	* We can find the current address in
3206	* mddev->curr_resync, but for recovery,
3207	* we need to convert that to several
3208	* virtual addresses.
3209	*/
3210	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
3211	end_reshape(conf);
3212	close_sync(conf);
3213	return 0;
3214	}
3215
3216	if (mddev->curr_resync < max_sector) { /* aborted */
3217	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
3218	md_bitmap_end_sync(mddev, offset: mddev->curr_resync,
3219	blocks: &sync_blocks);
3220	else for (i = 0; i < conf->geo.raid_disks; i++) {
3221	sector_t sect =
3222	raid10_find_virt(conf, sector: mddev->curr_resync, dev: i);
3223
3224	md_bitmap_end_sync(mddev, offset: sect, blocks: &sync_blocks);
3225	}
3226	} else {
3227	/* completed sync */
3228	if ((!mddev->bitmap || conf->fullsync)
3229	&& conf->have_replacement
3230	&& test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3231	/* Completed a full sync so the replacements
3232	* are now fully recovered.
3233	*/
3234	for (i = 0; i < conf->geo.raid_disks; i++) {
3235	struct md_rdev *rdev =
3236	conf->mirrors[i].replacement;
3237
3238	if (rdev)
3239	rdev->recovery_offset = MaxSector;
3240	}
3241	}
3242	conf->fullsync = 0;
3243	}
3244	if (md_bitmap_enabled(mddev, flush: false))
3245	mddev->bitmap_ops->close_sync(mddev);
3246	close_sync(conf);
3247	*skipped = 1;
3248	return sectors_skipped;
3249	}
3250
3251	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3252	return reshape_request(mddev, sector_nr, skipped);
3253
3254	if (chunks_skipped >= conf->geo.raid_disks) {
3255	pr_err("md/raid10:%s: %s fails\n", mdname(mddev),
3256	test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ? "resync" : "recovery");
3257	if (error_disk >= 0 &&
3258	!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3259	/*
3260	* recovery fails, set mirrors.recovery_disabled,
3261	* device shouldn't be added to there.
3262	*/
3263	conf->mirrors[error_disk].recovery_disabled =
3264	mddev->recovery_disabled;
3265	return 0;
3266	}
3267	/*
3268	* if there has been nothing to do on any drive,
3269	* then there is nothing to do at all.
3270	*/
3271	*skipped = 1;
3272	return (max_sector - sector_nr) + sectors_skipped;
3273	}
3274
3275	if (max_sector > mddev->resync_max)
3276	max_sector = mddev->resync_max; /* Don't do IO beyond here */
3277
3278	/* make sure whole request will fit in a chunk - if chunks
3279	* are meaningful
3280	*/
3281	if (conf->geo.near_copies < conf->geo.raid_disks &&
3282	max_sector > (sector_nr | chunk_mask))
3283	max_sector = (sector_nr | chunk_mask) + 1;
3284
3285	/*
3286	* If there is non-resync activity waiting for a turn, then let it
3287	* though before starting on this new sync request.
3288	*/
3289	if (conf->nr_waiting)
3290	schedule_timeout_uninterruptible(timeout: 1);
3291
3292	/* Again, very different code for resync and recovery.
3293	* Both must result in an r10bio with a list of bios that
3294	* have bi_end_io, bi_sector, bi_bdev set,
3295	* and bi_private set to the r10bio.
3296	* For recovery, we may actually create several r10bios
3297	* with 2 bios in each, that correspond to the bios in the main one.
3298	* In this case, the subordinate r10bios link back through a
3299	* borrowed master_bio pointer, and the counter in the master
3300	* includes a ref from each subordinate.
3301	*/
3302	/* First, we decide what to do and set ->bi_end_io
3303	* To end_sync_read if we want to read, and
3304	* end_sync_write if we will want to write.
3305	*/
3306
3307	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
3308	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3309	/* recovery... the complicated one */
3310	int j;
3311	r10_bio = NULL;
3312
3313	for (i = 0 ; i < conf->geo.raid_disks; i++) {
3314	bool still_degraded;
3315	struct r10bio *rb2;
3316	sector_t sect;
3317	bool must_sync;
3318	int any_working;
3319	struct raid10_info *mirror = &conf->mirrors[i];
3320	struct md_rdev *mrdev, *mreplace;
3321
3322	mrdev = mirror->rdev;
3323	mreplace = mirror->replacement;
3324
3325	if (mrdev && (test_bit(Faulty, &mrdev->flags) ||
3326	test_bit(In_sync, &mrdev->flags)))
3327	mrdev = NULL;
3328	if (mreplace && test_bit(Faulty, &mreplace->flags))
3329	mreplace = NULL;
3330
3331	if (!mrdev && !mreplace)
3332	continue;
3333
3334	still_degraded = false;
3335	/* want to reconstruct this device */
3336	rb2 = r10_bio;
3337	sect = raid10_find_virt(conf, sector: sector_nr, dev: i);
3338	if (sect >= mddev->resync_max_sectors)
3339	/* last stripe is not complete - don't
3340	* try to recover this sector.
3341	*/
3342	continue;
3343	/* Unless we are doing a full sync, or a replacement
3344	* we only need to recover the block if it is set in
3345	* the bitmap
3346	*/
3347	must_sync = md_bitmap_start_sync(mddev, offset: sect,
3348	blocks: &sync_blocks, degraded: true);
3349	if (sync_blocks < max_sync)
3350	max_sync = sync_blocks;
3351	if (!must_sync &&
3352	mreplace == NULL &&
3353	!conf->fullsync) {
3354	/* yep, skip the sync_blocks here, but don't assume
3355	* that there will never be anything to do here
3356	*/
3357	chunks_skipped = -1;
3358	continue;
3359	}
3360	if (mrdev)
3361	atomic_inc(v: &mrdev->nr_pending);
3362	if (mreplace)
3363	atomic_inc(v: &mreplace->nr_pending);
3364
3365	r10_bio = raid10_alloc_init_r10buf(conf);
3366	r10_bio->state = 0;
3367	raise_barrier(conf, force: rb2 != NULL);
3368	atomic_set(v: &r10_bio->remaining, i: 0);
3369
3370	r10_bio->master_bio = (struct bio*)rb2;
3371	if (rb2)
3372	atomic_inc(v: &rb2->remaining);
3373	r10_bio->mddev = mddev;
3374	set_bit(nr: R10BIO_IsRecover, addr: &r10_bio->state);
3375	r10_bio->sector = sect;
3376
3377	raid10_find_phys(conf, r10bio: r10_bio);
3378
3379	/* Need to check if the array will still be
3380	* degraded
3381	*/
3382	for (j = 0; j < conf->geo.raid_disks; j++) {
3383	struct md_rdev *rdev = conf->mirrors[j].rdev;
3384
3385	if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3386	still_degraded = false;
3387	break;
3388	}
3389	}
3390
3391	md_bitmap_start_sync(mddev, offset: sect, blocks: &sync_blocks,
3392	degraded: still_degraded);
3393	any_working = 0;
3394	for (j=0; j<conf->copies;j++) {
3395	int k;
3396	int d = r10_bio->devs[j].devnum;
3397	sector_t from_addr, to_addr;
3398	struct md_rdev *rdev = conf->mirrors[d].rdev;
3399	sector_t sector, first_bad;
3400	sector_t bad_sectors;
3401	if (!rdev ||
3402	!test_bit(In_sync, &rdev->flags))
3403	continue;
3404	/* This is where we read from */
3405	any_working = 1;
3406	sector = r10_bio->devs[j].addr;
3407
3408	if (is_badblock(rdev, s: sector, sectors: max_sync,
3409	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
3410	if (first_bad > sector)
3411	max_sync = first_bad - sector;
3412	else {
3413	bad_sectors -= (sector
3414	- first_bad);
3415	if (max_sync > bad_sectors)
3416	max_sync = bad_sectors;
3417	continue;
3418	}
3419	}
3420	bio = r10_bio->devs[0].bio;
3421	bio->bi_next = biolist;
3422	biolist = bio;
3423	bio->bi_end_io = end_sync_read;
3424	bio->bi_opf = REQ_OP_READ;
3425	if (test_bit(FailFast, &rdev->flags))
3426	bio->bi_opf |= MD_FAILFAST;
3427	from_addr = r10_bio->devs[j].addr;
3428	bio->bi_iter.bi_sector = from_addr +
3429	rdev->data_offset;
3430	bio_set_dev(bio, bdev: rdev->bdev);
3431	atomic_inc(v: &rdev->nr_pending);
3432	/* and we write to 'i' (if not in_sync) */
3433
3434	for (k=0; k<conf->copies; k++)
3435	if (r10_bio->devs[k].devnum == i)
3436	break;
3437	BUG_ON(k == conf->copies);
3438	to_addr = r10_bio->devs[k].addr;
3439	r10_bio->devs[0].devnum = d;
3440	r10_bio->devs[0].addr = from_addr;
3441	r10_bio->devs[1].devnum = i;
3442	r10_bio->devs[1].addr = to_addr;
3443
3444	if (mrdev) {
3445	bio = r10_bio->devs[1].bio;
3446	bio->bi_next = biolist;
3447	biolist = bio;
3448	bio->bi_end_io = end_sync_write;
3449	bio->bi_opf = REQ_OP_WRITE;
3450	bio->bi_iter.bi_sector = to_addr
3451	+ mrdev->data_offset;
3452	bio_set_dev(bio, bdev: mrdev->bdev);
3453	atomic_inc(v: &r10_bio->remaining);
3454	} else
3455	r10_bio->devs[1].bio->bi_end_io = NULL;
3456
3457	/* and maybe write to replacement */
3458	bio = r10_bio->devs[1].repl_bio;
3459	if (bio)
3460	bio->bi_end_io = NULL;
3461	/* Note: if replace is not NULL, then bio
3462	* cannot be NULL as r10buf_pool_alloc will
3463	* have allocated it.
3464	*/
3465	if (!mreplace)
3466	break;
3467	bio->bi_next = biolist;
3468	biolist = bio;
3469	bio->bi_end_io = end_sync_write;
3470	bio->bi_opf = REQ_OP_WRITE;
3471	bio->bi_iter.bi_sector = to_addr +
3472	mreplace->data_offset;
3473	bio_set_dev(bio, bdev: mreplace->bdev);
3474	atomic_inc(v: &r10_bio->remaining);
3475	break;
3476	}
3477	if (j == conf->copies) {
3478	/* Cannot recover, so abort the recovery or
3479	* record a bad block */
3480	if (any_working) {
3481	/* problem is that there are bad blocks
3482	* on other device(s)
3483	*/
3484	int k;
3485	for (k = 0; k < conf->copies; k++)
3486	if (r10_bio->devs[k].devnum == i)
3487	break;
3488	if (mrdev && !test_bit(In_sync,
3489	&mrdev->flags)
3490	&& !rdev_set_badblocks(
3491	rdev: mrdev,
3492	s: r10_bio->devs[k].addr,
3493	sectors: max_sync, is_new: 0))
3494	any_working = 0;
3495	if (mreplace &&
3496	!rdev_set_badblocks(
3497	rdev: mreplace,
3498	s: r10_bio->devs[k].addr,
3499	sectors: max_sync, is_new: 0))
3500	any_working = 0;
3501	}
3502	if (!any_working) {
3503	if (!test_and_set_bit(nr: MD_RECOVERY_INTR,
3504	addr: &mddev->recovery))
3505	pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
3506	mdname(mddev));
3507	mirror->recovery_disabled
3508	= mddev->recovery_disabled;
3509	} else {
3510	error_disk = i;
3511	}
3512	put_buf(r10_bio);
3513	if (rb2)
3514	atomic_dec(v: &rb2->remaining);
3515	r10_bio = rb2;
3516	if (mrdev)
3517	rdev_dec_pending(rdev: mrdev, mddev);
3518	if (mreplace)
3519	rdev_dec_pending(rdev: mreplace, mddev);
3520	break;
3521	}
3522	if (mrdev)
3523	rdev_dec_pending(rdev: mrdev, mddev);
3524	if (mreplace)
3525	rdev_dec_pending(rdev: mreplace, mddev);
3526	if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
3527	/* Only want this if there is elsewhere to
3528	* read from. 'j' is currently the first
3529	* readable copy.
3530	*/
3531	int targets = 1;
3532	for (; j < conf->copies; j++) {
3533	int d = r10_bio->devs[j].devnum;
3534	if (conf->mirrors[d].rdev &&
3535	test_bit(In_sync,
3536	&conf->mirrors[d].rdev->flags))
3537	targets++;
3538	}
3539	if (targets == 1)
3540	r10_bio->devs[0].bio->bi_opf
3541	&= ~MD_FAILFAST;
3542	}
3543	}
3544	if (biolist == NULL) {
3545	while (r10_bio) {
3546	struct r10bio *rb2 = r10_bio;
3547	r10_bio = (struct r10bio*) rb2->master_bio;
3548	rb2->master_bio = NULL;
3549	put_buf(r10_bio: rb2);
3550	}
3551	goto giveup;
3552	}
3553	} else {
3554	/* resync. Schedule a read for every block at this virt offset */
3555	int count = 0;
3556
3557	/*
3558	* Since curr_resync_completed could probably not update in
3559	* time, and we will set cluster_sync_low based on it.
3560	* Let's check against "sector_nr + 2 * RESYNC_SECTORS" for
3561	* safety reason, which ensures curr_resync_completed is
3562	* updated in bitmap_cond_end_sync.
3563	*/
3564	if (md_bitmap_enabled(mddev, flush: false))
3565	mddev->bitmap_ops->cond_end_sync(mddev, sector_nr,
3566	mddev_is_clustered(mddev) &&
3567	(sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
3568
3569	if (!md_bitmap_start_sync(mddev, offset: sector_nr, blocks: &sync_blocks,
3570	degraded: mddev->degraded) &&
3571	!conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
3572	&mddev->recovery)) {
3573	/* We can skip this block */
3574	*skipped = 1;
3575	return sync_blocks + sectors_skipped;
3576	}
3577	if (sync_blocks < max_sync)
3578	max_sync = sync_blocks;
3579	r10_bio = raid10_alloc_init_r10buf(conf);
3580	r10_bio->state = 0;
3581
3582	r10_bio->mddev = mddev;
3583	atomic_set(v: &r10_bio->remaining, i: 0);
3584	raise_barrier(conf, force: 0);
3585	conf->next_resync = sector_nr;
3586
3587	r10_bio->master_bio = NULL;
3588	r10_bio->sector = sector_nr;
3589	set_bit(nr: R10BIO_IsSync, addr: &r10_bio->state);
3590	raid10_find_phys(conf, r10bio: r10_bio);
3591	r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
3592
3593	for (i = 0; i < conf->copies; i++) {
3594	int d = r10_bio->devs[i].devnum;
3595	sector_t first_bad, sector;
3596	sector_t bad_sectors;
3597	struct md_rdev *rdev;
3598
3599	if (r10_bio->devs[i].repl_bio)
3600	r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3601
3602	bio = r10_bio->devs[i].bio;
3603	bio->bi_status = BLK_STS_IOERR;
3604	rdev = conf->mirrors[d].rdev;
3605	if (rdev == NULL || test_bit(Faulty, &rdev->flags))
3606	continue;
3607
3608	sector = r10_bio->devs[i].addr;
3609	if (is_badblock(rdev, s: sector, sectors: max_sync,
3610	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
3611	if (first_bad > sector)
3612	max_sync = first_bad - sector;
3613	else {
3614	bad_sectors -= (sector - first_bad);
3615	if (max_sync > bad_sectors)
3616	max_sync = bad_sectors;
3617	continue;
3618	}
3619	}
3620	atomic_inc(v: &rdev->nr_pending);
3621	atomic_inc(v: &r10_bio->remaining);
3622	bio->bi_next = biolist;
3623	biolist = bio;
3624	bio->bi_end_io = end_sync_read;
3625	bio->bi_opf = REQ_OP_READ;
3626	if (test_bit(FailFast, &rdev->flags))
3627	bio->bi_opf |= MD_FAILFAST;
3628	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3629	bio_set_dev(bio, bdev: rdev->bdev);
3630	count++;
3631
3632	rdev = conf->mirrors[d].replacement;
3633	if (rdev == NULL || test_bit(Faulty, &rdev->flags))
3634	continue;
3635
3636	atomic_inc(v: &rdev->nr_pending);
3637
3638	/* Need to set up for writing to the replacement */
3639	bio = r10_bio->devs[i].repl_bio;
3640	bio->bi_status = BLK_STS_IOERR;
3641
3642	sector = r10_bio->devs[i].addr;
3643	bio->bi_next = biolist;
3644	biolist = bio;
3645	bio->bi_end_io = end_sync_write;
3646	bio->bi_opf = REQ_OP_WRITE;
3647	if (test_bit(FailFast, &rdev->flags))
3648	bio->bi_opf |= MD_FAILFAST;
3649	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3650	bio_set_dev(bio, bdev: rdev->bdev);
3651	count++;
3652	}
3653
3654	if (count < 2) {
3655	for (i=0; i<conf->copies; i++) {
3656	int d = r10_bio->devs[i].devnum;
3657	if (r10_bio->devs[i].bio->bi_end_io)
3658	rdev_dec_pending(rdev: conf->mirrors[d].rdev,
3659	mddev);
3660	if (r10_bio->devs[i].repl_bio &&
3661	r10_bio->devs[i].repl_bio->bi_end_io)
3662	rdev_dec_pending(
3663	rdev: conf->mirrors[d].replacement,
3664	mddev);
3665	}
3666	put_buf(r10_bio);
3667	biolist = NULL;
3668	goto giveup;
3669	}
3670	}
3671
3672	nr_sectors = 0;
3673	if (sector_nr + max_sync < max_sector)
3674	max_sector = sector_nr + max_sync;
3675	do {
3676	struct page *page;
3677	int len = PAGE_SIZE;
3678	if (sector_nr + (len>>9) > max_sector)
3679	len = (max_sector - sector_nr) << 9;
3680	if (len == 0)
3681	break;
3682	for (bio= biolist ; bio ; bio=bio->bi_next) {
3683	struct resync_pages *rp = get_resync_pages(bio);
3684	page = resync_fetch_page(rp, idx: page_idx);
3685	if (WARN_ON(!bio_add_page(bio, page, len, 0))) {
3686	bio->bi_status = BLK_STS_RESOURCE;
3687	bio_endio(bio);
3688	goto giveup;
3689	}
3690	}
3691	nr_sectors += len>>9;
3692	sector_nr += len>>9;
3693	} while (++page_idx < RESYNC_PAGES);
3694	r10_bio->sectors = nr_sectors;
3695
3696	if (mddev_is_clustered(mddev) &&
3697	test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3698	/* It is resync not recovery */
3699	if (conf->cluster_sync_high < sector_nr + nr_sectors) {
3700	conf->cluster_sync_low = mddev->curr_resync_completed;
3701	raid10_set_cluster_sync_high(conf);
3702	/* Send resync message */
3703	mddev->cluster_ops->resync_info_update(mddev,
3704	conf->cluster_sync_low,
3705	conf->cluster_sync_high);
3706	}
3707	} else if (mddev_is_clustered(mddev)) {
3708	/* This is recovery not resync */
3709	sector_t sect_va1, sect_va2;
3710	bool broadcast_msg = false;
3711
3712	for (i = 0; i < conf->geo.raid_disks; i++) {
3713	/*
3714	* sector_nr is a device address for recovery, so we
3715	* need translate it to array address before compare
3716	* with cluster_sync_high.
3717	*/
3718	sect_va1 = raid10_find_virt(conf, sector: sector_nr, dev: i);
3719
3720	if (conf->cluster_sync_high < sect_va1 + nr_sectors) {
3721	broadcast_msg = true;
3722	/*
3723	* curr_resync_completed is similar as
3724	* sector_nr, so make the translation too.
3725	*/
3726	sect_va2 = raid10_find_virt(conf,
3727	sector: mddev->curr_resync_completed, dev: i);
3728
3729	if (conf->cluster_sync_low == 0 ||
3730	conf->cluster_sync_low > sect_va2)
3731	conf->cluster_sync_low = sect_va2;
3732	}
3733	}
3734	if (broadcast_msg) {
3735	raid10_set_cluster_sync_high(conf);
3736	mddev->cluster_ops->resync_info_update(mddev,
3737	conf->cluster_sync_low,
3738	conf->cluster_sync_high);
3739	}
3740	}
3741
3742	while (biolist) {
3743	bio = biolist;
3744	biolist = biolist->bi_next;
3745
3746	bio->bi_next = NULL;
3747	r10_bio = get_resync_r10bio(bio);
3748	r10_bio->sectors = nr_sectors;
3749
3750	if (bio->bi_end_io == end_sync_read) {
3751	bio->bi_status = 0;
3752	submit_bio_noacct(bio);
3753	}
3754	}
3755
3756	if (sectors_skipped)
3757	/* pretend they weren't skipped, it makes
3758	* no important difference in this case
3759	*/
3760	md_done_sync(mddev, blocks: sectors_skipped, ok: 1);
3761
3762	return sectors_skipped + nr_sectors;
3763	giveup:
3764	/* There is nowhere to write, so all non-sync
3765	* drives must be failed or in resync, all drives
3766	* have a bad block, so try the next chunk...
3767	*/
3768	if (sector_nr + max_sync < max_sector)
3769	max_sector = sector_nr + max_sync;
3770
3771	sectors_skipped += (max_sector - sector_nr);
3772	chunks_skipped ++;
3773	sector_nr = max_sector;
3774	goto skipped;
3775	}
3776
3777	static sector_t
3778	raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3779	{
3780	sector_t size;
3781	struct r10conf *conf = mddev->private;
3782
3783	if (!raid_disks)
3784	raid_disks = min(conf->geo.raid_disks,
3785	conf->prev.raid_disks);
3786	if (!sectors)
3787	sectors = conf->dev_sectors;
3788
3789	size = sectors >> conf->geo.chunk_shift;
3790	sector_div(size, conf->geo.far_copies);
3791	size = size * raid_disks;
3792	sector_div(size, conf->geo.near_copies);
3793
3794	return size << conf->geo.chunk_shift;
3795	}
3796
3797	static void calc_sectors(struct r10conf *conf, sector_t size)
3798	{
3799	/* Calculate the number of sectors-per-device that will
3800	* actually be used, and set conf->dev_sectors and
3801	* conf->stride
3802	*/
3803
3804	size = size >> conf->geo.chunk_shift;
3805	sector_div(size, conf->geo.far_copies);
3806	size = size * conf->geo.raid_disks;
3807	sector_div(size, conf->geo.near_copies);
3808	/* 'size' is now the number of chunks in the array */
3809	/* calculate "used chunks per device" */
3810	size = size * conf->copies;
3811
3812	/* We need to round up when dividing by raid_disks to
3813	* get the stride size.
3814	*/
3815	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3816
3817	conf->dev_sectors = size << conf->geo.chunk_shift;
3818
3819	if (conf->geo.far_offset)
3820	conf->geo.stride = 1 << conf->geo.chunk_shift;
3821	else {
3822	sector_div(size, conf->geo.far_copies);
3823	conf->geo.stride = size << conf->geo.chunk_shift;
3824	}
3825	}
3826
3827	enum geo_type {geo_new, geo_old, geo_start};
3828	static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
3829	{
3830	int nc, fc, fo;
3831	int layout, chunk, disks;
3832	switch (new) {
3833	case geo_old:
3834	layout = mddev->layout;
3835	chunk = mddev->chunk_sectors;
3836	disks = mddev->raid_disks - mddev->delta_disks;
3837	break;
3838	case geo_new:
3839	layout = mddev->new_layout;
3840	chunk = mddev->new_chunk_sectors;
3841	disks = mddev->raid_disks;
3842	break;
3843	default: /* avoid 'may be unused' warnings */
3844	case geo_start: /* new when starting reshape - raid_disks not
3845	* updated yet. */
3846	layout = mddev->new_layout;
3847	chunk = mddev->new_chunk_sectors;
3848	disks = mddev->raid_disks + mddev->delta_disks;
3849	break;
3850	}
3851	if (layout >> 19)
3852	return -1;
3853	if (chunk < (PAGE_SIZE >> 9) ||
3854	!is_power_of_2(n: chunk))
3855	return -2;
3856	nc = layout & 255;
3857	fc = (layout >> 8) & 255;
3858	fo = layout & (1<<16);
3859	geo->raid_disks = disks;
3860	geo->near_copies = nc;
3861	geo->far_copies = fc;
3862	geo->far_offset = fo;
3863	switch (layout >> 17) {
3864	case 0: /* original layout. simple but not always optimal */
3865	geo->far_set_size = disks;
3866	break;
3867	case 1: /* "improved" layout which was buggy. Hopefully no-one is
3868	* actually using this, but leave code here just in case.*/
3869	geo->far_set_size = disks/fc;
3870	WARN(geo->far_set_size < fc,
3871	"This RAID10 layout does not provide data safety - please backup and create new array\n");
3872	break;
3873	case 2: /* "improved" layout fixed to match documentation */
3874	geo->far_set_size = fc * nc;
3875	break;
3876	default: /* Not a valid layout */
3877	return -1;
3878	}
3879	geo->chunk_mask = chunk - 1;
3880	geo->chunk_shift = ffz(~chunk);
3881	return nc*fc;
3882	}
3883
3884	static void raid10_free_conf(struct r10conf *conf)
3885	{
3886	if (!conf)
3887	return;
3888
3889	mempool_exit(pool: &conf->r10bio_pool);
3890	kfree(objp: conf->mirrors);
3891	kfree(objp: conf->mirrors_old);
3892	kfree(objp: conf->mirrors_new);
3893	safe_put_page(p: conf->tmppage);
3894	bioset_exit(&conf->bio_split);
3895	kfree(objp: conf);
3896	}
3897
3898	static struct r10conf *setup_conf(struct mddev *mddev)
3899	{
3900	struct r10conf *conf = NULL;
3901	int err = -EINVAL;
3902	struct geom geo;
3903	int copies;
3904
3905	copies = setup_geo(geo: &geo, mddev, new: geo_new);
3906
3907	if (copies == -2) {
3908	pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
3909	mdname(mddev), PAGE_SIZE);
3910	goto out;
3911	}
3912
3913	if (copies < 2 || copies > mddev->raid_disks) {
3914	pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
3915	mdname(mddev), mddev->new_layout);
3916	goto out;
3917	}
3918
3919	err = -ENOMEM;
3920	conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
3921	if (!conf)
3922	goto out;
3923
3924	/* FIXME calc properly */
3925	conf->mirrors = kcalloc(mddev->raid_disks + max(0, -mddev->delta_disks),
3926	sizeof(struct raid10_info),
3927	GFP_KERNEL);
3928	if (!conf->mirrors)
3929	goto out;
3930
3931	conf->tmppage = alloc_page(GFP_KERNEL);
3932	if (!conf->tmppage)
3933	goto out;
3934
3935	conf->geo = geo;
3936	conf->copies = copies;
3937	err = mempool_init(&conf->r10bio_pool, NR_RAID_BIOS, r10bio_pool_alloc,
3938	rbio_pool_free, conf);
3939	if (err)
3940	goto out;
3941
3942	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, flags: 0);
3943	if (err)
3944	goto out;
3945
3946	calc_sectors(conf, size: mddev->dev_sectors);
3947	if (mddev->reshape_position == MaxSector) {
3948	conf->prev = conf->geo;
3949	conf->reshape_progress = MaxSector;
3950	} else {
3951	if (setup_geo(geo: &conf->prev, mddev, new: geo_old) != conf->copies) {
3952	err = -EINVAL;
3953	goto out;
3954	}
3955	conf->reshape_progress = mddev->reshape_position;
3956	if (conf->prev.far_offset)
3957	conf->prev.stride = 1 << conf->prev.chunk_shift;
3958	else
3959	/* far_copies must be 1 */
3960	conf->prev.stride = conf->dev_sectors;
3961	}
3962	conf->reshape_safe = conf->reshape_progress;
3963	spin_lock_init(&conf->device_lock);
3964	INIT_LIST_HEAD(list: &conf->retry_list);
3965	INIT_LIST_HEAD(list: &conf->bio_end_io_list);
3966
3967	seqlock_init(&conf->resync_lock);
3968	init_waitqueue_head(&conf->wait_barrier);
3969	atomic_set(v: &conf->nr_pending, i: 0);
3970
3971	err = -ENOMEM;
3972	rcu_assign_pointer(conf->thread,
3973	md_register_thread(raid10d, mddev, "raid10"));
3974	if (!conf->thread)
3975	goto out;
3976
3977	conf->mddev = mddev;
3978	return conf;
3979
3980	out:
3981	raid10_free_conf(conf);
3982	return ERR_PTR(error: err);
3983	}
3984
3985	static unsigned int raid10_nr_stripes(struct r10conf *conf)
3986	{
3987	unsigned int raid_disks = conf->geo.raid_disks;
3988
3989	if (conf->geo.raid_disks % conf->geo.near_copies)
3990	return raid_disks;
3991	return raid_disks / conf->geo.near_copies;
3992	}
3993
3994	static int raid10_set_queue_limits(struct mddev *mddev)
3995	{
3996	struct r10conf *conf = mddev->private;
3997	struct queue_limits lim;
3998	int err;
3999
4000	md_init_stacking_limits(lim: &lim);
4001	lim.max_write_zeroes_sectors = 0;
4002	lim.max_hw_wzeroes_unmap_sectors = 0;
4003	lim.logical_block_size = mddev->logical_block_size;
4004	lim.io_min = mddev->chunk_sectors << 9;
4005	lim.chunk_sectors = mddev->chunk_sectors;
4006	lim.io_opt = lim.io_min * raid10_nr_stripes(conf);
4007	lim.features |= BLK_FEAT_ATOMIC_WRITES;
4008	err = mddev_stack_rdev_limits(mddev, lim: &lim, MDDEV_STACK_INTEGRITY);
4009	if (err)
4010	return err;
4011	return queue_limits_set(q: mddev->gendisk->queue, lim: &lim);
4012	}
4013
4014	static int raid10_run(struct mddev *mddev)
4015	{
4016	struct r10conf *conf;
4017	int i, disk_idx;
4018	struct raid10_info *disk;
4019	struct md_rdev *rdev;
4020	sector_t size;
4021	sector_t min_offset_diff = 0;
4022	int first = 1;
4023	int ret = -EIO;
4024
4025	if (mddev->private == NULL) {
4026	conf = setup_conf(mddev);
4027	if (IS_ERR(ptr: conf))
4028	return PTR_ERR(ptr: conf);
4029	mddev->private = conf;
4030	}
4031	conf = mddev->private;
4032	if (!conf)
4033	goto out;
4034
4035	rcu_assign_pointer(mddev->thread, conf->thread);
4036	rcu_assign_pointer(conf->thread, NULL);
4037
4038	if (mddev_is_clustered(mddev: conf->mddev)) {
4039	int fc, fo;
4040
4041	fc = (mddev->layout >> 8) & 255;
4042	fo = mddev->layout & (1<<16);
4043	if (fc > 1 || fo > 0) {
4044	pr_err("only near layout is supported by clustered"
4045	" raid10\n");
4046	goto out_free_conf;
4047	}
4048	}
4049
4050	rdev_for_each(rdev, mddev) {
4051	long long diff;
4052
4053	disk_idx = rdev->raid_disk;
4054	if (disk_idx < 0)
4055	continue;
4056	if (disk_idx >= conf->geo.raid_disks &&
4057	disk_idx >= conf->prev.raid_disks)
4058	continue;
4059	disk = conf->mirrors + disk_idx;
4060
4061	if (test_bit(Replacement, &rdev->flags)) {
4062	if (disk->replacement)
4063	goto out_free_conf;
4064	disk->replacement = rdev;
4065	} else {
4066	if (disk->rdev)
4067	goto out_free_conf;
4068	disk->rdev = rdev;
4069	}
4070	diff = (rdev->new_data_offset - rdev->data_offset);
4071	if (!mddev->reshape_backwards)
4072	diff = -diff;
4073	if (diff < 0)
4074	diff = 0;
4075	if (first || diff < min_offset_diff)
4076	min_offset_diff = diff;
4077
4078	disk->head_position = 0;
4079	first = 0;
4080	}
4081
4082	if (!mddev_is_dm(mddev: conf->mddev)) {
4083	int err = raid10_set_queue_limits(mddev);
4084
4085	if (err) {
4086	ret = err;
4087	goto out_free_conf;
4088	}
4089	}
4090
4091	/* need to check that every block has at least one working mirror */
4092	if (!enough(conf, ignore: -1)) {
4093	pr_err("md/raid10:%s: not enough operational mirrors.\n",
4094	mdname(mddev));
4095	goto out_free_conf;
4096	}
4097
4098	if (conf->reshape_progress != MaxSector) {
4099	/* must ensure that shape change is supported */
4100	if (conf->geo.far_copies != 1 &&
4101	conf->geo.far_offset == 0)
4102	goto out_free_conf;
4103	if (conf->prev.far_copies != 1 &&
4104	conf->prev.far_offset == 0)
4105	goto out_free_conf;
4106	}
4107
4108	mddev->degraded = 0;
4109	for (i = 0;
4110	i < conf->geo.raid_disks
4111	|| i < conf->prev.raid_disks;
4112	i++) {
4113
4114	disk = conf->mirrors + i;
4115
4116	if (!disk->rdev && disk->replacement) {
4117	/* The replacement is all we have - use it */
4118	disk->rdev = disk->replacement;
4119	disk->replacement = NULL;
4120	clear_bit(nr: Replacement, addr: &disk->rdev->flags);
4121	}
4122
4123	if (!disk->rdev ||
4124	!test_bit(In_sync, &disk->rdev->flags)) {
4125	disk->head_position = 0;
4126	mddev->degraded++;
4127	if (disk->rdev &&
4128	disk->rdev->saved_raid_disk < 0)
4129	conf->fullsync = 1;
4130	}
4131
4132	if (disk->replacement &&
4133	!test_bit(In_sync, &disk->replacement->flags) &&
4134	disk->replacement->saved_raid_disk < 0) {
4135	conf->fullsync = 1;
4136	}
4137
4138	disk->recovery_disabled = mddev->recovery_disabled - 1;
4139	}
4140
4141	if (mddev->resync_offset != MaxSector)
4142	pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
4143	mdname(mddev));
4144	pr_info("md/raid10:%s: active with %d out of %d devices\n",
4145	mdname(mddev), conf->geo.raid_disks - mddev->degraded,
4146	conf->geo.raid_disks);
4147	/*
4148	* Ok, everything is just fine now
4149	*/
4150	mddev->dev_sectors = conf->dev_sectors;
4151	size = raid10_size(mddev, sectors: 0, raid_disks: 0);
4152	md_set_array_sectors(mddev, array_sectors: size);
4153	mddev->resync_max_sectors = size;
4154	set_bit(nr: MD_FAILFAST_SUPPORTED, addr: &mddev->flags);
4155
4156	if (md_integrity_register(mddev))
4157	goto out_free_conf;
4158
4159	if (conf->reshape_progress != MaxSector) {
4160	unsigned long before_length, after_length;
4161
4162	before_length = ((1 << conf->prev.chunk_shift) *
4163	conf->prev.far_copies);
4164	after_length = ((1 << conf->geo.chunk_shift) *
4165	conf->geo.far_copies);
4166
4167	if (max(before_length, after_length) > min_offset_diff) {
4168	/* This cannot work */
4169	pr_warn("md/raid10: offset difference not enough to continue reshape\n");
4170	goto out_free_conf;
4171	}
4172	conf->offset_diff = min_offset_diff;
4173
4174	clear_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
4175	clear_bit(nr: MD_RECOVERY_CHECK, addr: &mddev->recovery);
4176	set_bit(nr: MD_RECOVERY_RESHAPE, addr: &mddev->recovery);
4177	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
4178	}
4179
4180	return 0;
4181
4182	out_free_conf:
4183	md_unregister_thread(mddev, threadp: &mddev->thread);
4184	raid10_free_conf(conf);
4185	mddev->private = NULL;
4186	out:
4187	return ret;
4188	}
4189
4190	static void raid10_free(struct mddev *mddev, void *priv)
4191	{
4192	raid10_free_conf(conf: priv);
4193	}
4194
4195	static void raid10_quiesce(struct mddev *mddev, int quiesce)
4196	{
4197	struct r10conf *conf = mddev->private;
4198
4199	if (quiesce)
4200	raise_barrier(conf, force: 0);
4201	else
4202	lower_barrier(conf);
4203	}
4204
4205	static int raid10_resize(struct mddev *mddev, sector_t sectors)
4206	{
4207	/* Resize of 'far' arrays is not supported.
4208	* For 'near' and 'offset' arrays we can set the
4209	* number of sectors used to be an appropriate multiple
4210	* of the chunk size.
4211	* For 'offset', this is far_copies*chunksize.
4212	* For 'near' the multiplier is the LCM of
4213	* near_copies and raid_disks.
4214	* So if far_copies > 1 && !far_offset, fail.
4215	* Else find LCM(raid_disks, near_copy)*far_copies and
4216	* multiply by chunk_size. Then round to this number.
4217	* This is mostly done by raid10_size()
4218	*/
4219	struct r10conf *conf = mddev->private;
4220	sector_t oldsize, size;
4221
4222	if (mddev->reshape_position != MaxSector)
4223	return -EBUSY;
4224
4225	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
4226	return -EINVAL;
4227
4228	oldsize = raid10_size(mddev, sectors: 0, raid_disks: 0);
4229	size = raid10_size(mddev, sectors, raid_disks: 0);
4230	if (mddev->external_size &&
4231	mddev->array_sectors > size)
4232	return -EINVAL;
4233
4234	if (md_bitmap_enabled(mddev, flush: false)) {
4235	int ret = mddev->bitmap_ops->resize(mddev, size, 0);
4236
4237	if (ret)
4238	return ret;
4239	}
4240
4241	md_set_array_sectors(mddev, array_sectors: size);
4242	if (sectors > mddev->dev_sectors &&
4243	mddev->resync_offset > oldsize) {
4244	mddev->resync_offset = oldsize;
4245	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
4246	}
4247	calc_sectors(conf, size: sectors);
4248	mddev->dev_sectors = conf->dev_sectors;
4249	mddev->resync_max_sectors = size;
4250	return 0;
4251	}
4252
4253	static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
4254	{
4255	struct md_rdev *rdev;
4256	struct r10conf *conf;
4257
4258	if (mddev->degraded > 0) {
4259	pr_warn("md/raid10:%s: Error: degraded raid0!\n",
4260	mdname(mddev));
4261	return ERR_PTR(error: -EINVAL);
4262	}
4263	sector_div(size, devs);
4264
4265	/* Set new parameters */
4266	mddev->new_level = 10;
4267	/* new layout: far_copies = 1, near_copies = 2 */
4268	mddev->new_layout = (1<<8) + 2;
4269	mddev->new_chunk_sectors = mddev->chunk_sectors;
4270	mddev->delta_disks = mddev->raid_disks;
4271	mddev->raid_disks *= 2;
4272	/* make sure it will be not marked as dirty */
4273	mddev->resync_offset = MaxSector;
4274	mddev->dev_sectors = size;
4275
4276	conf = setup_conf(mddev);
4277	if (!IS_ERR(ptr: conf)) {
4278	rdev_for_each(rdev, mddev)
4279	if (rdev->raid_disk >= 0) {
4280	rdev->new_raid_disk = rdev->raid_disk * 2;
4281	rdev->sectors = size;
4282	}
4283	}
4284
4285	return conf;
4286	}
4287
4288	static void *raid10_takeover(struct mddev *mddev)
4289	{
4290	struct r0conf *raid0_conf;
4291
4292	/* raid10 can take over:
4293	* raid0 - providing it has only two drives
4294	*/
4295	if (mddev->level == 0) {
4296	/* for raid0 takeover only one zone is supported */
4297	raid0_conf = mddev->private;
4298	if (raid0_conf->nr_strip_zones > 1) {
4299	pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
4300	mdname(mddev));
4301	return ERR_PTR(error: -EINVAL);
4302	}
4303	return raid10_takeover_raid0(mddev,
4304	size: raid0_conf->strip_zone->zone_end,
4305	devs: raid0_conf->strip_zone->nb_dev);
4306	}
4307	return ERR_PTR(error: -EINVAL);
4308	}
4309
4310	static int raid10_check_reshape(struct mddev *mddev)
4311	{
4312	/* Called when there is a request to change
4313	* - layout (to ->new_layout)
4314	* - chunk size (to ->new_chunk_sectors)
4315	* - raid_disks (by delta_disks)
4316	* or when trying to restart a reshape that was ongoing.
4317	*
4318	* We need to validate the request and possibly allocate
4319	* space if that might be an issue later.
4320	*
4321	* Currently we reject any reshape of a 'far' mode array,
4322	* allow chunk size to change if new is generally acceptable,
4323	* allow raid_disks to increase, and allow
4324	* a switch between 'near' mode and 'offset' mode.
4325	*/
4326	struct r10conf *conf = mddev->private;
4327	struct geom geo;
4328
4329	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
4330	return -EINVAL;
4331
4332	if (setup_geo(geo: &geo, mddev, new: geo_start) != conf->copies)
4333	/* mustn't change number of copies */
4334	return -EINVAL;
4335	if (geo.far_copies > 1 && !geo.far_offset)
4336	/* Cannot switch to 'far' mode */
4337	return -EINVAL;
4338
4339	if (mddev->array_sectors & geo.chunk_mask)
4340	/* not factor of array size */
4341	return -EINVAL;
4342
4343	if (!enough(conf, ignore: -1))
4344	return -EINVAL;
4345
4346	kfree(objp: conf->mirrors_new);
4347	conf->mirrors_new = NULL;
4348	if (mddev->delta_disks > 0) {
4349	/* allocate new 'mirrors' list */
4350	conf->mirrors_new =
4351	kcalloc(mddev->raid_disks + mddev->delta_disks,
4352	sizeof(struct raid10_info),
4353	GFP_KERNEL);
4354	if (!conf->mirrors_new)
4355	return -ENOMEM;
4356	}
4357	return 0;
4358	}
4359
4360	/*
4361	* Need to check if array has failed when deciding whether to:
4362	* - start an array
4363	* - remove non-faulty devices
4364	* - add a spare
4365	* - allow a reshape
4366	* This determination is simple when no reshape is happening.
4367	* However if there is a reshape, we need to carefully check
4368	* both the before and after sections.
4369	* This is because some failed devices may only affect one
4370	* of the two sections, and some non-in_sync devices may
4371	* be insync in the section most affected by failed devices.
4372	*/
4373	static int calc_degraded(struct r10conf *conf)
4374	{
4375	int degraded, degraded2;
4376	int i;
4377
4378	degraded = 0;
4379	/* 'prev' section first */
4380	for (i = 0; i < conf->prev.raid_disks; i++) {
4381	struct md_rdev *rdev = conf->mirrors[i].rdev;
4382
4383	if (!rdev || test_bit(Faulty, &rdev->flags))
4384	degraded++;
4385	else if (!test_bit(In_sync, &rdev->flags))
4386	/* When we can reduce the number of devices in
4387	* an array, this might not contribute to
4388	* 'degraded'. It does now.
4389	*/
4390	degraded++;
4391	}
4392	if (conf->geo.raid_disks == conf->prev.raid_disks)
4393	return degraded;
4394	degraded2 = 0;
4395	for (i = 0; i < conf->geo.raid_disks; i++) {
4396	struct md_rdev *rdev = conf->mirrors[i].rdev;
4397
4398	if (!rdev || test_bit(Faulty, &rdev->flags))
4399	degraded2++;
4400	else if (!test_bit(In_sync, &rdev->flags)) {
4401	/* If reshape is increasing the number of devices,
4402	* this section has already been recovered, so
4403	* it doesn't contribute to degraded.
4404	* else it does.
4405	*/
4406	if (conf->geo.raid_disks <= conf->prev.raid_disks)
4407	degraded2++;
4408	}
4409	}
4410	if (degraded2 > degraded)
4411	return degraded2;
4412	return degraded;
4413	}
4414
4415	static int raid10_start_reshape(struct mddev *mddev)
4416	{
4417	/* A 'reshape' has been requested. This commits
4418	* the various 'new' fields and sets MD_RECOVER_RESHAPE
4419	* This also checks if there are enough spares and adds them
4420	* to the array.
4421	* We currently require enough spares to make the final
4422	* array non-degraded. We also require that the difference
4423	* between old and new data_offset - on each device - is
4424	* enough that we never risk over-writing.
4425	*/
4426
4427	unsigned long before_length, after_length;
4428	sector_t min_offset_diff = 0;
4429	int first = 1;
4430	struct geom new;
4431	struct r10conf *conf = mddev->private;
4432	struct md_rdev *rdev;
4433	int spares = 0;
4434	int ret;
4435
4436	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
4437	return -EBUSY;
4438
4439	if (setup_geo(geo: &new, mddev, new: geo_start) != conf->copies)
4440	return -EINVAL;
4441
4442	before_length = ((1 << conf->prev.chunk_shift) *
4443	conf->prev.far_copies);
4444	after_length = ((1 << conf->geo.chunk_shift) *
4445	conf->geo.far_copies);
4446
4447	rdev_for_each(rdev, mddev) {
4448	if (!test_bit(In_sync, &rdev->flags)
4449	&& !test_bit(Faulty, &rdev->flags))
4450	spares++;
4451	if (rdev->raid_disk >= 0) {
4452	long long diff = (rdev->new_data_offset
4453	- rdev->data_offset);
4454	if (!mddev->reshape_backwards)
4455	diff = -diff;
4456	if (diff < 0)
4457	diff = 0;
4458	if (first || diff < min_offset_diff)
4459	min_offset_diff = diff;
4460	first = 0;
4461	}
4462	}
4463
4464	if (max(before_length, after_length) > min_offset_diff)
4465	return -EINVAL;
4466
4467	if (spares < mddev->delta_disks)
4468	return -EINVAL;
4469
4470	conf->offset_diff = min_offset_diff;
4471	spin_lock_irq(lock: &conf->device_lock);
4472	if (conf->mirrors_new) {
4473	memcpy(conf->mirrors_new, conf->mirrors,
4474	sizeof(struct raid10_info)*conf->prev.raid_disks);
4475	smp_mb();
4476	kfree(objp: conf->mirrors_old);
4477	conf->mirrors_old = conf->mirrors;
4478	conf->mirrors = conf->mirrors_new;
4479	conf->mirrors_new = NULL;
4480	}
4481	setup_geo(geo: &conf->geo, mddev, new: geo_start);
4482	smp_mb();
4483	if (mddev->reshape_backwards) {
4484	sector_t size = raid10_size(mddev, sectors: 0, raid_disks: 0);
4485	if (size < mddev->array_sectors) {
4486	spin_unlock_irq(lock: &conf->device_lock);
4487	pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
4488	mdname(mddev));
4489	return -EINVAL;
4490	}
4491	mddev->resync_max_sectors = size;
4492	conf->reshape_progress = size;
4493	} else
4494	conf->reshape_progress = 0;
4495	conf->reshape_safe = conf->reshape_progress;
4496	spin_unlock_irq(lock: &conf->device_lock);
4497
4498	if (mddev->delta_disks && mddev->bitmap) {
4499	struct mdp_superblock_1 *sb = NULL;
4500	sector_t oldsize, newsize;
4501
4502	oldsize = raid10_size(mddev, sectors: 0, raid_disks: 0);
4503	newsize = raid10_size(mddev, sectors: 0, raid_disks: conf->geo.raid_disks);
4504
4505	if (!mddev_is_clustered(mddev) &&
4506	md_bitmap_enabled(mddev, flush: false)) {
4507	ret = mddev->bitmap_ops->resize(mddev, newsize, 0);
4508	if (ret)
4509	goto abort;
4510	else
4511	goto out;
4512	}
4513
4514	rdev_for_each(rdev, mddev) {
4515	if (rdev->raid_disk > -1 &&
4516	!test_bit(Faulty, &rdev->flags))
4517	sb = page_address(rdev->sb_page);
4518	}
4519
4520	/*
4521	* some node is already performing reshape, and no need to
4522	* call bitmap_ops->resize again since it should be called when
4523	* receiving BITMAP_RESIZE msg
4524	*/
4525	if ((sb && (le32_to_cpu(sb->feature_map) &
4526	MD_FEATURE_RESHAPE_ACTIVE)) || (oldsize == newsize))
4527	goto out;
4528
4529	/* cluster can't be setup without bitmap */
4530	ret = mddev->bitmap_ops->resize(mddev, newsize, 0);
4531	if (ret)
4532	goto abort;
4533
4534	ret = mddev->cluster_ops->resize_bitmaps(mddev, newsize, oldsize);
4535	if (ret) {
4536	mddev->bitmap_ops->resize(mddev, oldsize, 0);
4537	goto abort;
4538	}
4539	}
4540	out:
4541	if (mddev->delta_disks > 0) {
4542	rdev_for_each(rdev, mddev)
4543	if (rdev->raid_disk < 0 &&
4544	!test_bit(Faulty, &rdev->flags)) {
4545	if (raid10_add_disk(mddev, rdev) == 0) {
4546	if (rdev->raid_disk >=
4547	conf->prev.raid_disks)
4548	set_bit(nr: In_sync, addr: &rdev->flags);
4549	else
4550	rdev->recovery_offset = 0;
4551
4552	/* Failure here is OK */
4553	sysfs_link_rdev(mddev, rdev);
4554	}
4555	} else if (rdev->raid_disk >= conf->prev.raid_disks
4556	&& !test_bit(Faulty, &rdev->flags)) {
4557	/* This is a spare that was manually added */
4558	set_bit(nr: In_sync, addr: &rdev->flags);
4559	}
4560	}
4561	/* When a reshape changes the number of devices,
4562	* ->degraded is measured against the larger of the
4563	* pre and post numbers.
4564	*/
4565	spin_lock_irq(lock: &conf->device_lock);
4566	mddev->degraded = calc_degraded(conf);
4567	spin_unlock_irq(lock: &conf->device_lock);
4568	mddev->raid_disks = conf->geo.raid_disks;
4569	mddev->reshape_position = conf->reshape_progress;
4570	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
4571
4572	clear_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
4573	clear_bit(nr: MD_RECOVERY_CHECK, addr: &mddev->recovery);
4574	clear_bit(nr: MD_RECOVERY_DONE, addr: &mddev->recovery);
4575	set_bit(nr: MD_RECOVERY_RESHAPE, addr: &mddev->recovery);
4576	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
4577	conf->reshape_checkpoint = jiffies;
4578	md_new_event();
4579	return 0;
4580
4581	abort:
4582	mddev->recovery = 0;
4583	spin_lock_irq(lock: &conf->device_lock);
4584	conf->geo = conf->prev;
4585	mddev->raid_disks = conf->geo.raid_disks;
4586	rdev_for_each(rdev, mddev)
4587	rdev->new_data_offset = rdev->data_offset;
4588	smp_wmb();
4589	conf->reshape_progress = MaxSector;
4590	conf->reshape_safe = MaxSector;
4591	mddev->reshape_position = MaxSector;
4592	spin_unlock_irq(lock: &conf->device_lock);
4593	return ret;
4594	}
4595
4596	/* Calculate the last device-address that could contain
4597	* any block from the chunk that includes the array-address 's'
4598	* and report the next address.
4599	* i.e. the address returned will be chunk-aligned and after
4600	* any data that is in the chunk containing 's'.
4601	*/
4602	static sector_t last_dev_address(sector_t s, struct geom *geo)
4603	{
4604	s = (s | geo->chunk_mask) + 1;
4605	s >>= geo->chunk_shift;
4606	s *= geo->near_copies;
4607	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
4608	s *= geo->far_copies;
4609	s <<= geo->chunk_shift;
4610	return s;
4611	}
4612
4613	/* Calculate the first device-address that could contain
4614	* any block from the chunk that includes the array-address 's'.
4615	* This too will be the start of a chunk
4616	*/
4617	static sector_t first_dev_address(sector_t s, struct geom *geo)
4618	{
4619	s >>= geo->chunk_shift;
4620	s *= geo->near_copies;
4621	sector_div(s, geo->raid_disks);
4622	s *= geo->far_copies;
4623	s <<= geo->chunk_shift;
4624	return s;
4625	}
4626
4627	static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
4628	int *skipped)
4629	{
4630	/* We simply copy at most one chunk (smallest of old and new)
4631	* at a time, possibly less if that exceeds RESYNC_PAGES,
4632	* or we hit a bad block or something.
4633	* This might mean we pause for normal IO in the middle of
4634	* a chunk, but that is not a problem as mddev->reshape_position
4635	* can record any location.
4636	*
4637	* If we will want to write to a location that isn't
4638	* yet recorded as 'safe' (i.e. in metadata on disk) then
4639	* we need to flush all reshape requests and update the metadata.
4640	*
4641	* When reshaping forwards (e.g. to more devices), we interpret
4642	* 'safe' as the earliest block which might not have been copied
4643	* down yet. We divide this by previous stripe size and multiply
4644	* by previous stripe length to get lowest device offset that we
4645	* cannot write to yet.
4646	* We interpret 'sector_nr' as an address that we want to write to.
4647	* From this we use last_device_address() to find where we might
4648	* write to, and first_device_address on the 'safe' position.
4649	* If this 'next' write position is after the 'safe' position,
4650	* we must update the metadata to increase the 'safe' position.
4651	*
4652	* When reshaping backwards, we round in the opposite direction
4653	* and perform the reverse test: next write position must not be
4654	* less than current safe position.
4655	*
4656	* In all this the minimum difference in data offsets
4657	* (conf->offset_diff - always positive) allows a bit of slack,
4658	* so next can be after 'safe', but not by more than offset_diff
4659	*
4660	* We need to prepare all the bios here before we start any IO
4661	* to ensure the size we choose is acceptable to all devices.
4662	* The means one for each copy for write-out and an extra one for
4663	* read-in.
4664	* We store the read-in bio in ->master_bio and the others in
4665	* ->devs[x].bio and ->devs[x].repl_bio.
4666	*/
4667	struct r10conf *conf = mddev->private;
4668	struct r10bio *r10_bio;
4669	sector_t next, safe, last;
4670	int max_sectors;
4671	int nr_sectors;
4672	int s;
4673	struct md_rdev *rdev;
4674	int need_flush = 0;
4675	struct bio *blist;
4676	struct bio *bio, *read_bio;
4677	int sectors_done = 0;
4678	struct page **pages;
4679
4680	if (sector_nr == 0) {
4681	/* If restarting in the middle, skip the initial sectors */
4682	if (mddev->reshape_backwards &&
4683	conf->reshape_progress < raid10_size(mddev, sectors: 0, raid_disks: 0)) {
4684	sector_nr = (raid10_size(mddev, sectors: 0, raid_disks: 0)
4685	- conf->reshape_progress);
4686	} else if (!mddev->reshape_backwards &&
4687	conf->reshape_progress > 0)
4688	sector_nr = conf->reshape_progress;
4689	if (sector_nr) {
4690	mddev->curr_resync_completed = sector_nr;
4691	sysfs_notify_dirent_safe(sd: mddev->sysfs_completed);
4692	*skipped = 1;
4693	return sector_nr;
4694	}
4695	}
4696
4697	/* We don't use sector_nr to track where we are up to
4698	* as that doesn't work well for ->reshape_backwards.
4699	* So just use ->reshape_progress.
4700	*/
4701	if (mddev->reshape_backwards) {
4702	/* 'next' is the earliest device address that we might
4703	* write to for this chunk in the new layout
4704	*/
4705	next = first_dev_address(s: conf->reshape_progress - 1,
4706	geo: &conf->geo);
4707
4708	/* 'safe' is the last device address that we might read from
4709	* in the old layout after a restart
4710	*/
4711	safe = last_dev_address(s: conf->reshape_safe - 1,
4712	geo: &conf->prev);
4713
4714	if (next + conf->offset_diff < safe)
4715	need_flush = 1;
4716
4717	last = conf->reshape_progress - 1;
4718	sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
4719	& conf->prev.chunk_mask);
4720	if (sector_nr + RESYNC_SECTORS < last)
4721	sector_nr = last + 1 - RESYNC_SECTORS;
4722	} else {
4723	/* 'next' is after the last device address that we
4724	* might write to for this chunk in the new layout
4725	*/
4726	next = last_dev_address(s: conf->reshape_progress, geo: &conf->geo);
4727
4728	/* 'safe' is the earliest device address that we might
4729	* read from in the old layout after a restart
4730	*/
4731	safe = first_dev_address(s: conf->reshape_safe, geo: &conf->prev);
4732
4733	/* Need to update metadata if 'next' might be beyond 'safe'
4734	* as that would possibly corrupt data
4735	*/
4736	if (next > safe + conf->offset_diff)
4737	need_flush = 1;
4738
4739	sector_nr = conf->reshape_progress;
4740	last = sector_nr | (conf->geo.chunk_mask
4741	& conf->prev.chunk_mask);
4742
4743	if (sector_nr + RESYNC_SECTORS <= last)
4744	last = sector_nr + RESYNC_SECTORS - 1;
4745	}
4746
4747	if (need_flush ||
4748	time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4749	/* Need to update reshape_position in metadata */
4750	wait_barrier(conf, nowait: false);
4751	mddev->reshape_position = conf->reshape_progress;
4752	if (mddev->reshape_backwards)
4753	mddev->curr_resync_completed = raid10_size(mddev, sectors: 0, raid_disks: 0)
4754	- conf->reshape_progress;
4755	else
4756	mddev->curr_resync_completed = conf->reshape_progress;
4757	conf->reshape_checkpoint = jiffies;
4758	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
4759	md_wakeup_thread(mddev->thread);
4760	wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
4761	test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4762	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
4763	allow_barrier(conf);
4764	return sectors_done;
4765	}
4766	conf->reshape_safe = mddev->reshape_position;
4767	allow_barrier(conf);
4768	}
4769
4770	raise_barrier(conf, force: 0);
4771	read_more:
4772	/* Now schedule reads for blocks from sector_nr to last */
4773	r10_bio = raid10_alloc_init_r10buf(conf);
4774	r10_bio->state = 0;
4775	raise_barrier(conf, force: 1);
4776	atomic_set(v: &r10_bio->remaining, i: 0);
4777	r10_bio->mddev = mddev;
4778	r10_bio->sector = sector_nr;
4779	set_bit(nr: R10BIO_IsReshape, addr: &r10_bio->state);
4780	r10_bio->sectors = last - sector_nr + 1;
4781	rdev = read_balance(conf, r10_bio, max_sectors: &max_sectors);
4782	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));
4783
4784	if (!rdev) {
4785	/* Cannot read from here, so need to record bad blocks
4786	* on all the target devices.
4787	*/
4788	// FIXME
4789	mempool_free(element: r10_bio, pool: &conf->r10buf_pool);
4790	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
4791	return sectors_done;
4792	}
4793
4794	read_bio = bio_alloc_bioset(bdev: rdev->bdev, RESYNC_PAGES, opf: REQ_OP_READ,
4795	GFP_KERNEL, bs: &mddev->bio_set);
4796	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4797	+ rdev->data_offset);
4798	read_bio->bi_private = r10_bio;
4799	read_bio->bi_end_io = end_reshape_read;
4800	r10_bio->master_bio = read_bio;
4801	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
4802
4803	/*
4804	* Broadcast RESYNC message to other nodes, so all nodes would not
4805	* write to the region to avoid conflict.
4806	*/
4807	if (mddev_is_clustered(mddev) && conf->cluster_sync_high <= sector_nr) {
4808	struct mdp_superblock_1 *sb = NULL;
4809	int sb_reshape_pos = 0;
4810
4811	conf->cluster_sync_low = sector_nr;
4812	conf->cluster_sync_high = sector_nr + CLUSTER_RESYNC_WINDOW_SECTORS;
4813	sb = page_address(rdev->sb_page);
4814	if (sb) {
4815	sb_reshape_pos = le64_to_cpu(sb->reshape_position);
4816	/*
4817	* Set cluster_sync_low again if next address for array
4818	* reshape is less than cluster_sync_low. Since we can't
4819	* update cluster_sync_low until it has finished reshape.
4820	*/
4821	if (sb_reshape_pos < conf->cluster_sync_low)
4822	conf->cluster_sync_low = sb_reshape_pos;
4823	}
4824
4825	mddev->cluster_ops->resync_info_update(mddev, conf->cluster_sync_low,
4826	conf->cluster_sync_high);
4827	}
4828
4829	/* Now find the locations in the new layout */
4830	__raid10_find_phys(geo: &conf->geo, r10bio: r10_bio);
4831
4832	blist = read_bio;
4833	read_bio->bi_next = NULL;
4834
4835	for (s = 0; s < conf->copies*2; s++) {
4836	struct bio *b;
4837	int d = r10_bio->devs[s/2].devnum;
4838	struct md_rdev *rdev2;
4839	if (s&1) {
4840	rdev2 = conf->mirrors[d].replacement;
4841	b = r10_bio->devs[s/2].repl_bio;
4842	} else {
4843	rdev2 = conf->mirrors[d].rdev;
4844	b = r10_bio->devs[s/2].bio;
4845	}
4846	if (!rdev2 || test_bit(Faulty, &rdev2->flags))
4847	continue;
4848
4849	bio_set_dev(bio: b, bdev: rdev2->bdev);
4850	b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
4851	rdev2->new_data_offset;
4852	b->bi_end_io = end_reshape_write;
4853	b->bi_opf = REQ_OP_WRITE;
4854	b->bi_next = blist;
4855	blist = b;
4856	}
4857
4858	/* Now add as many pages as possible to all of these bios. */
4859
4860	nr_sectors = 0;
4861	pages = get_resync_pages(bio: r10_bio->devs[0].bio)->pages;
4862	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4863	struct page *page = pages[s / (PAGE_SIZE >> 9)];
4864	int len = (max_sectors - s) << 9;
4865	if (len > PAGE_SIZE)
4866	len = PAGE_SIZE;
4867	for (bio = blist; bio ; bio = bio->bi_next) {
4868	if (WARN_ON(!bio_add_page(bio, page, len, 0))) {
4869	bio->bi_status = BLK_STS_RESOURCE;
4870	bio_endio(bio);
4871	return sectors_done;
4872	}
4873	}
4874	sector_nr += len >> 9;
4875	nr_sectors += len >> 9;
4876	}
4877	r10_bio->sectors = nr_sectors;
4878
4879	/* Now submit the read */
4880	atomic_inc(v: &r10_bio->remaining);
4881	read_bio->bi_next = NULL;
4882	submit_bio_noacct(bio: read_bio);
4883	sectors_done += nr_sectors;
4884	if (sector_nr <= last)
4885	goto read_more;
4886
4887	lower_barrier(conf);
4888
4889	/* Now that we have done the whole section we can
4890	* update reshape_progress
4891	*/
4892	if (mddev->reshape_backwards)
4893	conf->reshape_progress -= sectors_done;
4894	else
4895	conf->reshape_progress += sectors_done;
4896
4897	return sectors_done;
4898	}
4899
4900	static void end_reshape_request(struct r10bio *r10_bio);
4901	static int handle_reshape_read_error(struct mddev *mddev,
4902	struct r10bio *r10_bio);
4903	static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
4904	{
4905	/* Reshape read completed. Hopefully we have a block
4906	* to write out.
4907	* If we got a read error then we do sync 1-page reads from
4908	* elsewhere until we find the data - or give up.
4909	*/
4910	struct r10conf *conf = mddev->private;
4911	int s;
4912
4913	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
4914	if (handle_reshape_read_error(mddev, r10_bio) < 0) {
4915	/* Reshape has been aborted */
4916	md_done_sync(mddev, blocks: r10_bio->sectors, ok: 0);
4917	return;
4918	}
4919
4920	/* We definitely have the data in the pages, schedule the
4921	* writes.
4922	*/
4923	atomic_set(v: &r10_bio->remaining, i: 1);
4924	for (s = 0; s < conf->copies*2; s++) {
4925	struct bio *b;
4926	int d = r10_bio->devs[s/2].devnum;
4927	struct md_rdev *rdev;
4928	if (s&1) {
4929	rdev = conf->mirrors[d].replacement;
4930	b = r10_bio->devs[s/2].repl_bio;
4931	} else {
4932	rdev = conf->mirrors[d].rdev;
4933	b = r10_bio->devs[s/2].bio;
4934	}
4935	if (!rdev || test_bit(Faulty, &rdev->flags))
4936	continue;
4937
4938	atomic_inc(v: &rdev->nr_pending);
4939	atomic_inc(v: &r10_bio->remaining);
4940	b->bi_next = NULL;
4941	submit_bio_noacct(bio: b);
4942	}
4943	end_reshape_request(r10_bio);
4944	}
4945
4946	static void end_reshape(struct r10conf *conf)
4947	{
4948	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
4949	return;
4950
4951	spin_lock_irq(lock: &conf->device_lock);
4952	conf->prev = conf->geo;
4953	md_finish_reshape(mddev: conf->mddev);
4954	smp_wmb();
4955	conf->reshape_progress = MaxSector;
4956	conf->reshape_safe = MaxSector;
4957	spin_unlock_irq(lock: &conf->device_lock);
4958
4959	mddev_update_io_opt(mddev: conf->mddev, nr_stripes: raid10_nr_stripes(conf));
4960	conf->fullsync = 0;
4961	}
4962
4963	static void raid10_update_reshape_pos(struct mddev *mddev)
4964	{
4965	struct r10conf *conf = mddev->private;
4966	sector_t lo, hi;
4967
4968	mddev->cluster_ops->resync_info_get(mddev, &lo, &hi);
4969	if (((mddev->reshape_position <= hi) && (mddev->reshape_position >= lo))
4970	|| mddev->reshape_position == MaxSector)
4971	conf->reshape_progress = mddev->reshape_position;
4972	else
4973	WARN_ON_ONCE(1);
4974	}
4975
4976	static int handle_reshape_read_error(struct mddev *mddev,
4977	struct r10bio *r10_bio)
4978	{
4979	/* Use sync reads to get the blocks from somewhere else */
4980	int sectors = r10_bio->sectors;
4981	struct r10conf *conf = mddev->private;
4982	struct r10bio *r10b;
4983	int slot = 0;
4984	int idx = 0;
4985	struct page **pages;
4986
4987	r10b = kmalloc(struct_size(r10b, devs, conf->copies), GFP_NOIO);
4988	if (!r10b) {
4989	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
4990	return -ENOMEM;
4991	}
4992
4993	/* reshape IOs share pages from .devs[0].bio */
4994	pages = get_resync_pages(bio: r10_bio->devs[0].bio)->pages;
4995
4996	r10b->sector = r10_bio->sector;
4997	__raid10_find_phys(geo: &conf->prev, r10bio: r10b);
4998
4999	while (sectors) {
5000	int s = sectors;
5001	int success = 0;
5002	int first_slot = slot;
5003
5004	if (s > (PAGE_SIZE >> 9))
5005	s = PAGE_SIZE >> 9;
5006
5007	while (!success) {
5008	int d = r10b->devs[slot].devnum;
5009	struct md_rdev *rdev = conf->mirrors[d].rdev;
5010	sector_t addr;
5011	if (rdev == NULL ||
5012	test_bit(Faulty, &rdev->flags) ||
5013	!test_bit(In_sync, &rdev->flags))
5014	goto failed;
5015
5016	addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
5017	atomic_inc(v: &rdev->nr_pending);
5018	success = sync_page_io(rdev,
5019	sector: addr,
5020	size: s << 9,
5021	page: pages[idx],
5022	opf: REQ_OP_READ, metadata_op: false);
5023	rdev_dec_pending(rdev, mddev);
5024	if (success)
5025	break;
5026	failed:
5027	slot++;
5028	if (slot >= conf->copies)
5029	slot = 0;
5030	if (slot == first_slot)
5031	break;
5032	}
5033	if (!success) {
5034	/* couldn't read this block, must give up */
5035	set_bit(nr: MD_RECOVERY_INTR,
5036	addr: &mddev->recovery);
5037	kfree(objp: r10b);
5038	return -EIO;
5039	}
5040	sectors -= s;
5041	idx++;
5042	}
5043	kfree(objp: r10b);
5044	return 0;
5045	}
5046
5047	static void end_reshape_write(struct bio *bio)
5048	{
5049	struct r10bio *r10_bio = get_resync_r10bio(bio);
5050	struct mddev *mddev = r10_bio->mddev;
5051	struct r10conf *conf = mddev->private;
5052	int d;
5053	int slot;
5054	int repl;
5055	struct md_rdev *rdev = NULL;
5056
5057	d = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
5058	rdev = repl ? conf->mirrors[d].replacement :
5059	conf->mirrors[d].rdev;
5060
5061	if (bio->bi_status) {
5062	/* FIXME should record badblock */
5063	md_error(mddev, rdev);
5064	}
5065
5066	rdev_dec_pending(rdev, mddev);
5067	end_reshape_request(r10_bio);
5068	}
5069
5070	static void end_reshape_request(struct r10bio *r10_bio)
5071	{
5072	if (!atomic_dec_and_test(v: &r10_bio->remaining))
5073	return;
5074	md_done_sync(mddev: r10_bio->mddev, blocks: r10_bio->sectors, ok: 1);
5075	bio_put(r10_bio->master_bio);
5076	put_buf(r10_bio);
5077	}
5078
5079	static void raid10_finish_reshape(struct mddev *mddev)
5080	{
5081	struct r10conf *conf = mddev->private;
5082
5083	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5084	return;
5085
5086	if (mddev->delta_disks > 0) {
5087	if (mddev->resync_offset > mddev->resync_max_sectors) {
5088	mddev->resync_offset = mddev->resync_max_sectors;
5089	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
5090	}
5091	mddev->resync_max_sectors = mddev->array_sectors;
5092	} else {
5093	int d;
5094	for (d = conf->geo.raid_disks ;
5095	d < conf->geo.raid_disks - mddev->delta_disks;
5096	d++) {
5097	struct md_rdev *rdev = conf->mirrors[d].rdev;
5098	if (rdev)
5099	clear_bit(nr: In_sync, addr: &rdev->flags);
5100	rdev = conf->mirrors[d].replacement;
5101	if (rdev)
5102	clear_bit(nr: In_sync, addr: &rdev->flags);
5103	}
5104	}
5105	mddev->layout = mddev->new_layout;
5106	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
5107	mddev->reshape_position = MaxSector;
5108	mddev->delta_disks = 0;
5109	mddev->reshape_backwards = 0;
5110	}
5111
5112	static struct md_personality raid10_personality =
5113	{
5114	.head = {
5115	.type = MD_PERSONALITY,
5116	.id = ID_RAID10,
5117	.name = "raid10",
5118	.owner = THIS_MODULE,
5119	},
5120
5121	.make_request = raid10_make_request,
5122	.run = raid10_run,
5123	.free = raid10_free,
5124	.status = raid10_status,
5125	.error_handler = raid10_error,
5126	.hot_add_disk = raid10_add_disk,
5127	.hot_remove_disk= raid10_remove_disk,
5128	.spare_active = raid10_spare_active,
5129	.sync_request = raid10_sync_request,
5130	.quiesce = raid10_quiesce,
5131	.size = raid10_size,
5132	.resize = raid10_resize,
5133	.takeover = raid10_takeover,
5134	.check_reshape = raid10_check_reshape,
5135	.start_reshape = raid10_start_reshape,
5136	.finish_reshape = raid10_finish_reshape,
5137	.update_reshape_pos = raid10_update_reshape_pos,
5138	};
5139
5140	static int __init raid10_init(void)
5141	{
5142	return register_md_submodule(msh: &raid10_personality.head);
5143	}
5144
5145	static void __exit raid10_exit(void)
5146	{
5147	unregister_md_submodule(msh: &raid10_personality.head);
5148	}
5149
5150	module_init(raid10_init);
5151	module_exit(raid10_exit);
5152	MODULE_LICENSE("GPL");
5153	MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
5154	MODULE_ALIAS("md-personality-9"); /* RAID10 */
5155	MODULE_ALIAS("md-raid10");
5156	MODULE_ALIAS("md-level-10");
5157