1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2012 Red Hat, Inc.
4	*
5	* Author: Mikulas Patocka <mpatocka@redhat.com>
6	*
7	* Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
8	*
9	* In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
10	* default prefetch value. Data are read in "prefetch_cluster" chunks from the
11	* hash device. Setting this greatly improves performance when data and hash
12	* are on the same disk on different partitions on devices with poor random
13	* access behavior.
14	*/
15
16	#include "dm-verity.h"
17	#include "dm-verity-fec.h"
18	#include "dm-verity-verify-sig.h"
19	#include "dm-audit.h"
20	#include <linux/module.h>
21	#include <linux/reboot.h>
22	#include <linux/string.h>
23	#include <linux/jump_label.h>
24	#include <linux/security.h>
25
26	#define DM_MSG_PREFIX "verity"
27
28	#define DM_VERITY_ENV_LENGTH 42
29	#define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"
30
31	#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
32	#define DM_VERITY_USE_BH_DEFAULT_BYTES 8192
33
34	#define DM_VERITY_MAX_CORRUPTED_ERRS 100
35
36	#define DM_VERITY_OPT_LOGGING "ignore_corruption"
37	#define DM_VERITY_OPT_RESTART "restart_on_corruption"
38	#define DM_VERITY_OPT_PANIC "panic_on_corruption"
39	#define DM_VERITY_OPT_ERROR_RESTART "restart_on_error"
40	#define DM_VERITY_OPT_ERROR_PANIC "panic_on_error"
41	#define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"
42	#define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once"
43	#define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet"
44
45	#define DM_VERITY_OPTS_MAX (5 + DM_VERITY_OPTS_FEC + \
46	DM_VERITY_ROOT_HASH_VERIFICATION_OPTS)
47
48	static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
49
50	module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);
51
52	static unsigned int dm_verity_use_bh_bytes[4] = {
53	DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_NONE
54	DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_RT
55	DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_BE
56	0 // IOPRIO_CLASS_IDLE
57	};
58
59	module_param_array_named(use_bh_bytes, dm_verity_use_bh_bytes, uint, NULL, 0644);
60
61	static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);
62
63	struct dm_verity_prefetch_work {
64	struct work_struct work;
65	struct dm_verity *v;
66	unsigned short ioprio;
67	sector_t block;
68	unsigned int n_blocks;
69	};
70
71	/*
72	* Auxiliary structure appended to each dm-bufio buffer. If the value
73	* hash_verified is nonzero, hash of the block has been verified.
74	*
75	* The variable hash_verified is set to 0 when allocating the buffer, then
76	* it can be changed to 1 and it is never reset to 0 again.
77	*
78	* There is no lock around this value, a race condition can at worst cause
79	* that multiple processes verify the hash of the same buffer simultaneously
80	* and write 1 to hash_verified simultaneously.
81	* This condition is harmless, so we don't need locking.
82	*/
83	struct buffer_aux {
84	int hash_verified;
85	};
86
87	/*
88	* Initialize struct buffer_aux for a freshly created buffer.
89	*/
90	static void dm_bufio_alloc_callback(struct dm_buffer *buf)
91	{
92	struct buffer_aux *aux = dm_bufio_get_aux_data(b: buf);
93
94	aux->hash_verified = 0;
95	}
96
97	/*
98	* Translate input sector number to the sector number on the target device.
99	*/
100	static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
101	{
102	return dm_target_offset(v->ti, bi_sector);
103	}
104
105	/*
106	* Return hash position of a specified block at a specified tree level
107	* (0 is the lowest level).
108	* The lowest "hash_per_block_bits"-bits of the result denote hash position
109	* inside a hash block. The remaining bits denote location of the hash block.
110	*/
111	static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
112	int level)
113	{
114	return block >> (level * v->hash_per_block_bits);
115	}
116
117	int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
118	const u8 *data, size_t len, u8 *digest)
119	{
120	struct shash_desc *desc;
121	int r;
122
123	if (likely(v->use_sha256_lib)) {
124	struct sha256_ctx *ctx = &io->hash_ctx.sha256;
125
126	/*
127	* Fast path using SHA-256 library. This is enabled only for
128	* verity version 1, where the salt is at the beginning.
129	*/
130	*ctx = *v->initial_hashstate.sha256;
131	sha256_update(ctx, data, len);
132	sha256_final(ctx, out: digest);
133	return 0;
134	}
135
136	desc = &io->hash_ctx.shash;
137	desc->tfm = v->shash_tfm;
138	if (unlikely(v->initial_hashstate.shash == NULL)) {
139	/* Version 0: salt at end */
140	r = crypto_shash_init(desc) ?:
141	crypto_shash_update(desc, data, len) ?:
142	crypto_shash_update(desc, data: v->salt, len: v->salt_size) ?:
143	crypto_shash_final(desc, out: digest);
144	} else {
145	/* Version 1: salt at beginning */
146	r = crypto_shash_import(desc, in: v->initial_hashstate.shash) ?:
147	crypto_shash_finup(desc, data, len, out: digest);
148	}
149	if (unlikely(r))
150	DMERR("Error hashing block: %d", r);
151	return r;
152	}
153
154	static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
155	sector_t *hash_block, unsigned int *offset)
156	{
157	sector_t position = verity_position_at_level(v, block, level);
158	unsigned int idx;
159
160	*hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
161
162	if (!offset)
163	return;
164
165	idx = position & ((1 << v->hash_per_block_bits) - 1);
166	if (!v->version)
167	*offset = idx * v->digest_size;
168	else
169	*offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);
170	}
171
172	/*
173	* Handle verification errors.
174	*/
175	static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
176	unsigned long long block)
177	{
178	char verity_env[DM_VERITY_ENV_LENGTH];
179	char *envp[] = { verity_env, NULL };
180	const char *type_str = "";
181	struct mapped_device *md = dm_table_get_md(t: v->ti->table);
182
183	/* Corruption should be visible in device status in all modes */
184	v->hash_failed = true;
185
186	if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)
187	goto out;
188
189	v->corrupted_errs++;
190
191	switch (type) {
192	case DM_VERITY_BLOCK_TYPE_DATA:
193	type_str = "data";
194	break;
195	case DM_VERITY_BLOCK_TYPE_METADATA:
196	type_str = "metadata";
197	break;
198	default:
199	BUG();
200	}
201
202	DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name,
203	type_str, block);
204
205	if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) {
206	DMERR("%s: reached maximum errors", v->data_dev->name);
207	dm_audit_log_target(DM_MSG_PREFIX, op: "max-corrupted-errors", ti: v->ti, result: 0);
208	}
209
210	snprintf(buf: verity_env, DM_VERITY_ENV_LENGTH, fmt: "%s=%d,%llu",
211	DM_VERITY_ENV_VAR_NAME, type, block);
212
213	kobject_uevent_env(kobj: &disk_to_dev(dm_disk(md))->kobj, action: KOBJ_CHANGE, envp);
214
215	out:
216	if (v->mode == DM_VERITY_MODE_LOGGING)
217	return 0;
218
219	if (v->mode == DM_VERITY_MODE_RESTART)
220	kernel_restart(cmd: "dm-verity device corrupted");
221
222	if (v->mode == DM_VERITY_MODE_PANIC)
223	panic(fmt: "dm-verity device corrupted");
224
225	return 1;
226	}
227
228	/*
229	* Verify hash of a metadata block pertaining to the specified data block
230	* ("block" argument) at a specified level ("level" argument).
231	*
232	* On successful return, want_digest contains the hash value for a lower tree
233	* level or for the data block (if we're at the lowest level).
234	*
235	* If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
236	* If "skip_unverified" is false, unverified buffer is hashed and verified
237	* against current value of want_digest.
238	*/
239	static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
240	sector_t block, int level, bool skip_unverified,
241	u8 *want_digest)
242	{
243	struct dm_buffer *buf;
244	struct buffer_aux *aux;
245	u8 *data;
246	int r;
247	sector_t hash_block;
248	unsigned int offset;
249	struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
250
251	verity_hash_at_level(v, block, level, hash_block: &hash_block, offset: &offset);
252
253	if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
254	data = dm_bufio_get(c: v->bufio, block: hash_block, bp: &buf);
255	if (IS_ERR_OR_NULL(ptr: data)) {
256	/*
257	* In tasklet and the hash was not in the bufio cache.
258	* Return early and resume execution from a work-queue
259	* to read the hash from disk.
260	*/
261	return -EAGAIN;
262	}
263	} else {
264	data = dm_bufio_read_with_ioprio(c: v->bufio, block: hash_block,
265	bp: &buf, ioprio: bio->bi_ioprio);
266	}
267
268	if (IS_ERR(ptr: data)) {
269	if (skip_unverified)
270	return 1;
271	r = PTR_ERR(ptr: data);
272	data = dm_bufio_new(c: v->bufio, block: hash_block, bp: &buf);
273	if (IS_ERR(ptr: data))
274	return r;
275	if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_METADATA,
276	want_digest, block: hash_block, dest: data) == 0) {
277	aux = dm_bufio_get_aux_data(b: buf);
278	aux->hash_verified = 1;
279	goto release_ok;
280	} else {
281	dm_bufio_release(b: buf);
282	dm_bufio_forget(c: v->bufio, block: hash_block);
283	return r;
284	}
285	}
286
287	aux = dm_bufio_get_aux_data(b: buf);
288
289	if (!aux->hash_verified) {
290	if (skip_unverified) {
291	r = 1;
292	goto release_ret_r;
293	}
294
295	r = verity_hash(v, io, data, len: 1 << v->hash_dev_block_bits,
296	digest: io->tmp_digest);
297	if (unlikely(r < 0))
298	goto release_ret_r;
299
300	if (likely(memcmp(io->tmp_digest, want_digest,
301	v->digest_size) == 0))
302	aux->hash_verified = 1;
303	else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
304	/*
305	* Error handling code (FEC included) cannot be run in a
306	* tasklet since it may sleep, so fallback to work-queue.
307	*/
308	r = -EAGAIN;
309	goto release_ret_r;
310	} else if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_METADATA,
311	want_digest, block: hash_block, dest: data) == 0)
312	aux->hash_verified = 1;
313	else if (verity_handle_err(v,
314	type: DM_VERITY_BLOCK_TYPE_METADATA,
315	block: hash_block)) {
316	struct bio *bio;
317	io->had_mismatch = true;
318	bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
319	dm_audit_log_bio(DM_MSG_PREFIX, op: "verify-metadata", bio,
320	sector: block, result: 0);
321	r = -EIO;
322	goto release_ret_r;
323	}
324	}
325
326	release_ok:
327	data += offset;
328	memcpy(want_digest, data, v->digest_size);
329	r = 0;
330
331	release_ret_r:
332	dm_bufio_release(b: buf);
333	return r;
334	}
335
336	/*
337	* Find a hash for a given block, write it to digest and verify the integrity
338	* of the hash tree if necessary.
339	*/
340	int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
341	sector_t block, u8 *digest, bool *is_zero)
342	{
343	int r = 0, i;
344
345	if (likely(v->levels)) {
346	/*
347	* First, we try to get the requested hash for
348	* the current block. If the hash block itself is
349	* verified, zero is returned. If it isn't, this
350	* function returns 1 and we fall back to whole
351	* chain verification.
352	*/
353	r = verity_verify_level(v, io, block, level: 0, skip_unverified: true, want_digest: digest);
354	if (likely(r <= 0))
355	goto out;
356	}
357
358	memcpy(digest, v->root_digest, v->digest_size);
359
360	for (i = v->levels - 1; i >= 0; i--) {
361	r = verity_verify_level(v, io, block, level: i, skip_unverified: false, want_digest: digest);
362	if (unlikely(r))
363	goto out;
364	}
365	out:
366	if (!r && v->zero_digest)
367	*is_zero = !memcmp(p: v->zero_digest, q: digest, size: v->digest_size);
368	else
369	*is_zero = false;
370
371	return r;
372	}
373
374	static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io,
375	const u8 *want_digest, sector_t cur_block,
376	u8 *dest)
377	{
378	struct page *page;
379	void *buffer;
380	int r;
381	struct dm_io_request io_req;
382	struct dm_io_region io_loc;
383
384	page = mempool_alloc(&v->recheck_pool, GFP_NOIO);
385	buffer = page_to_virt(page);
386
387	io_req.bi_opf = REQ_OP_READ;
388	io_req.mem.type = DM_IO_KMEM;
389	io_req.mem.ptr.addr = buffer;
390	io_req.notify.fn = NULL;
391	io_req.client = v->io;
392	io_loc.bdev = v->data_dev->bdev;
393	io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);
394	io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);
395	r = dm_io(io_req: &io_req, num_regions: 1, region: &io_loc, NULL, IOPRIO_DEFAULT);
396	if (unlikely(r))
397	goto free_ret;
398
399	r = verity_hash(v, io, data: buffer, len: 1 << v->data_dev_block_bits,
400	digest: io->tmp_digest);
401	if (unlikely(r))
402	goto free_ret;
403
404	if (memcmp(p: io->tmp_digest, q: want_digest, size: v->digest_size)) {
405	r = -EIO;
406	goto free_ret;
407	}
408
409	memcpy(dest, buffer, 1 << v->data_dev_block_bits);
410	r = 0;
411	free_ret:
412	mempool_free(element: page, pool: &v->recheck_pool);
413
414	return r;
415	}
416
417	static int verity_handle_data_hash_mismatch(struct dm_verity *v,
418	struct dm_verity_io *io,
419	struct bio *bio,
420	struct pending_block *block)
421	{
422	const u8 *want_digest = block->want_digest;
423	sector_t blkno = block->blkno;
424	u8 *data = block->data;
425
426	if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
427	/*
428	* Error handling code (FEC included) cannot be run in the
429	* BH workqueue, so fallback to a standard workqueue.
430	*/
431	return -EAGAIN;
432	}
433	if (verity_recheck(v, io, want_digest, cur_block: blkno, dest: data) == 0) {
434	if (v->validated_blocks)
435	set_bit(nr: blkno, addr: v->validated_blocks);
436	return 0;
437	}
438	#if defined(CONFIG_DM_VERITY_FEC)
439	if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_DATA, want_digest,
440	block: blkno, dest: data) == 0)
441	return 0;
442	#endif
443	if (bio->bi_status)
444	return -EIO; /* Error correction failed; Just return error */
445
446	if (verity_handle_err(v, type: DM_VERITY_BLOCK_TYPE_DATA, block: blkno)) {
447	io->had_mismatch = true;
448	dm_audit_log_bio(DM_MSG_PREFIX, op: "verify-data", bio, sector: blkno, result: 0);
449	return -EIO;
450	}
451	return 0;
452	}
453
454	static void verity_clear_pending_blocks(struct dm_verity_io *io)
455	{
456	int i;
457
458	for (i = io->num_pending - 1; i >= 0; i--) {
459	kunmap_local(io->pending_blocks[i].data);
460	io->pending_blocks[i].data = NULL;
461	}
462	io->num_pending = 0;
463	}
464
465	static int verity_verify_pending_blocks(struct dm_verity *v,
466	struct dm_verity_io *io,
467	struct bio *bio)
468	{
469	const unsigned int block_size = 1 << v->data_dev_block_bits;
470	int i, r;
471
472	if (io->num_pending == 2) {
473	/* num_pending == 2 implies that the algorithm is SHA-256 */
474	sha256_finup_2x(ctx: v->initial_hashstate.sha256,
475	data1: io->pending_blocks[0].data,
476	data2: io->pending_blocks[1].data, len: block_size,
477	out1: io->pending_blocks[0].real_digest,
478	out2: io->pending_blocks[1].real_digest);
479	} else {
480	for (i = 0; i < io->num_pending; i++) {
481	r = verity_hash(v, io, data: io->pending_blocks[i].data,
482	len: block_size,
483	digest: io->pending_blocks[i].real_digest);
484	if (unlikely(r))
485	return r;
486	}
487	}
488
489	for (i = 0; i < io->num_pending; i++) {
490	struct pending_block *block = &io->pending_blocks[i];
491
492	if (likely(memcmp(block->real_digest, block->want_digest,
493	v->digest_size) == 0)) {
494	if (v->validated_blocks)
495	set_bit(nr: block->blkno, addr: v->validated_blocks);
496	} else {
497	r = verity_handle_data_hash_mismatch(v, io, bio, block);
498	if (unlikely(r))
499	return r;
500	}
501	}
502	verity_clear_pending_blocks(io);
503	return 0;
504	}
505
506	/*
507	* Verify one "dm_verity_io" structure.
508	*/
509	static int verity_verify_io(struct dm_verity_io *io)
510	{
511	struct dm_verity *v = io->v;
512	const unsigned int block_size = 1 << v->data_dev_block_bits;
513	const int max_pending = v->use_sha256_finup_2x ? 2 : 1;
514	struct bvec_iter iter_copy;
515	struct bvec_iter *iter;
516	struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
517	unsigned int b;
518	int r;
519
520	io->num_pending = 0;
521
522	if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
523	/*
524	* Copy the iterator in case we need to restart
525	* verification in a work-queue.
526	*/
527	iter_copy = io->iter;
528	iter = &iter_copy;
529	} else
530	iter = &io->iter;
531
532	for (b = 0; b < io->n_blocks;
533	b++, bio_advance_iter(bio, iter, bytes: block_size)) {
534	sector_t blkno = io->block + b;
535	struct pending_block *block;
536	bool is_zero;
537	struct bio_vec bv;
538	void *data;
539
540	if (v->validated_blocks && bio->bi_status == BLK_STS_OK &&
541	likely(test_bit(blkno, v->validated_blocks)))
542	continue;
543
544	block = &io->pending_blocks[io->num_pending];
545
546	r = verity_hash_for_block(v, io, block: blkno, digest: block->want_digest,
547	is_zero: &is_zero);
548	if (unlikely(r < 0))
549	goto error;
550
551	bv = bio_iter_iovec(bio, *iter);
552	if (unlikely(bv.bv_len < block_size)) {
553	/*
554	* Data block spans pages. This should not happen,
555	* since dm-verity sets dma_alignment to the data block
556	* size minus 1, and dm-verity also doesn't allow the
557	* data block size to be greater than PAGE_SIZE.
558	*/
559	DMERR_LIMIT("unaligned io (data block spans pages)");
560	r = -EIO;
561	goto error;
562	}
563
564	data = bvec_kmap_local(bvec: &bv);
565
566	if (is_zero) {
567	/*
568	* If we expect a zero block, don't validate, just
569	* return zeros.
570	*/
571	memset(data, 0, block_size);
572	kunmap_local(data);
573	continue;
574	}
575	block->data = data;
576	block->blkno = blkno;
577	if (++io->num_pending == max_pending) {
578	r = verity_verify_pending_blocks(v, io, bio);
579	if (unlikely(r))
580	goto error;
581	}
582	}
583
584	if (io->num_pending) {
585	r = verity_verify_pending_blocks(v, io, bio);
586	if (unlikely(r))
587	goto error;
588	}
589
590	return 0;
591
592	error:
593	verity_clear_pending_blocks(io);
594	return r;
595	}
596
597	/*
598	* Skip verity work in response to I/O error when system is shutting down.
599	*/
600	static inline bool verity_is_system_shutting_down(void)
601	{
602	return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF
603	|| system_state == SYSTEM_RESTART;
604	}
605
606	static void restart_io_error(struct work_struct *w)
607	{
608	kernel_restart(cmd: "dm-verity device has I/O error");
609	}
610
611	/*
612	* End one "io" structure with a given error.
613	*/
614	static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
615	{
616	struct dm_verity *v = io->v;
617	struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
618
619	bio->bi_end_io = io->orig_bi_end_io;
620	bio->bi_status = status;
621
622	if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)
623	verity_fec_finish_io(io);
624
625	if (unlikely(status != BLK_STS_OK) &&
626	unlikely(!(bio->bi_opf & REQ_RAHEAD)) &&
627	!io->had_mismatch &&
628	!verity_is_system_shutting_down()) {
629	if (v->error_mode == DM_VERITY_MODE_PANIC) {
630	panic(fmt: "dm-verity device has I/O error");
631	}
632	if (v->error_mode == DM_VERITY_MODE_RESTART) {
633	static DECLARE_WORK(restart_work, restart_io_error);
634	queue_work(wq: v->verify_wq, work: &restart_work);
635	/*
636	* We deliberately don't call bio_endio here, because
637	* the machine will be restarted anyway.
638	*/
639	return;
640	}
641	}
642
643	bio_endio(bio);
644	}
645
646	static void verity_work(struct work_struct *w)
647	{
648	struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
649
650	io->in_bh = false;
651
652	verity_finish_io(io, status: errno_to_blk_status(errno: verity_verify_io(io)));
653	}
654
655	static void verity_bh_work(struct work_struct *w)
656	{
657	struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);
658	int err;
659
660	io->in_bh = true;
661	err = verity_verify_io(io);
662	if (err == -EAGAIN || err == -ENOMEM) {
663	/* fallback to retrying with work-queue */
664	INIT_WORK(&io->work, verity_work);
665	queue_work(wq: io->v->verify_wq, work: &io->work);
666	return;
667	}
668
669	verity_finish_io(io, status: errno_to_blk_status(errno: err));
670	}
671
672	static inline bool verity_use_bh(unsigned int bytes, unsigned short ioprio)
673	{
674	return ioprio <= IOPRIO_CLASS_IDLE &&
675	bytes <= READ_ONCE(dm_verity_use_bh_bytes[ioprio]) &&
676	!need_resched();
677	}
678
679	static void verity_end_io(struct bio *bio)
680	{
681	struct dm_verity_io *io = bio->bi_private;
682	unsigned short ioprio = IOPRIO_PRIO_CLASS(bio->bi_ioprio);
683	unsigned int bytes = io->n_blocks << io->v->data_dev_block_bits;
684
685	if (bio->bi_status &&
686	(!verity_fec_is_enabled(v: io->v) ||
687	verity_is_system_shutting_down() ||
688	(bio->bi_opf & REQ_RAHEAD))) {
689	verity_finish_io(io, status: bio->bi_status);
690	return;
691	}
692
693	if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq &&
694	verity_use_bh(bytes, ioprio)) {
695	if (in_hardirq() || irqs_disabled()) {
696	INIT_WORK(&io->bh_work, verity_bh_work);
697	queue_work(wq: system_bh_wq, work: &io->bh_work);
698	} else {
699	verity_bh_work(w: &io->bh_work);
700	}
701	} else {
702	INIT_WORK(&io->work, verity_work);
703	queue_work(wq: io->v->verify_wq, work: &io->work);
704	}
705	}
706
707	/*
708	* Prefetch buffers for the specified io.
709	* The root buffer is not prefetched, it is assumed that it will be cached
710	* all the time.
711	*/
712	static void verity_prefetch_io(struct work_struct *work)
713	{
714	struct dm_verity_prefetch_work *pw =
715	container_of(work, struct dm_verity_prefetch_work, work);
716	struct dm_verity *v = pw->v;
717	int i;
718
719	for (i = v->levels - 2; i >= 0; i--) {
720	sector_t hash_block_start;
721	sector_t hash_block_end;
722
723	verity_hash_at_level(v, block: pw->block, level: i, hash_block: &hash_block_start, NULL);
724	verity_hash_at_level(v, block: pw->block + pw->n_blocks - 1, level: i, hash_block: &hash_block_end, NULL);
725
726	if (!i) {
727	unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster);
728
729	cluster >>= v->data_dev_block_bits;
730	if (unlikely(!cluster))
731	goto no_prefetch_cluster;
732
733	if (unlikely(cluster & (cluster - 1)))
734	cluster = 1 << __fls(word: cluster);
735
736	hash_block_start &= ~(sector_t)(cluster - 1);
737	hash_block_end |= cluster - 1;
738	if (unlikely(hash_block_end >= v->hash_blocks))
739	hash_block_end = v->hash_blocks - 1;
740	}
741	no_prefetch_cluster:
742	dm_bufio_prefetch_with_ioprio(c: v->bufio, block: hash_block_start,
743	n_blocks: hash_block_end - hash_block_start + 1,
744	ioprio: pw->ioprio);
745	}
746
747	kfree(objp: pw);
748	}
749
750	static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io,
751	unsigned short ioprio)
752	{
753	sector_t block = io->block;
754	unsigned int n_blocks = io->n_blocks;
755	struct dm_verity_prefetch_work *pw;
756
757	if (v->validated_blocks) {
758	while (n_blocks && test_bit(block, v->validated_blocks)) {
759	block++;
760	n_blocks--;
761	}
762	while (n_blocks && test_bit(block + n_blocks - 1,
763	v->validated_blocks))
764	n_blocks--;
765	if (!n_blocks)
766	return;
767	}
768
769	pw = kmalloc(sizeof(struct dm_verity_prefetch_work),
770	GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
771
772	if (!pw)
773	return;
774
775	INIT_WORK(&pw->work, verity_prefetch_io);
776	pw->v = v;
777	pw->block = block;
778	pw->n_blocks = n_blocks;
779	pw->ioprio = ioprio;
780	queue_work(wq: v->verify_wq, work: &pw->work);
781	}
782
783	/*
784	* Bio map function. It allocates dm_verity_io structure and bio vector and
785	* fills them. Then it issues prefetches and the I/O.
786	*/
787	static int verity_map(struct dm_target *ti, struct bio *bio)
788	{
789	struct dm_verity *v = ti->private;
790	struct dm_verity_io *io;
791
792	bio_set_dev(bio, bdev: v->data_dev->bdev);
793	bio->bi_iter.bi_sector = verity_map_sector(v, bi_sector: bio->bi_iter.bi_sector);
794
795	if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) &
796	((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
797	DMERR_LIMIT("unaligned io");
798	return DM_MAPIO_KILL;
799	}
800
801	if (bio_end_sector(bio) >>
802	(v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
803	DMERR_LIMIT("io out of range");
804	return DM_MAPIO_KILL;
805	}
806
807	if (bio_data_dir(bio) == WRITE)
808	return DM_MAPIO_KILL;
809
810	io = dm_per_bio_data(bio, data_size: ti->per_io_data_size);
811	io->v = v;
812	io->orig_bi_end_io = bio->bi_end_io;
813	io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
814	io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;
815	io->had_mismatch = false;
816
817	bio->bi_end_io = verity_end_io;
818	bio->bi_private = io;
819	io->iter = bio->bi_iter;
820
821	verity_fec_init_io(io);
822
823	verity_submit_prefetch(v, io, ioprio: bio->bi_ioprio);
824
825	submit_bio_noacct(bio);
826
827	return DM_MAPIO_SUBMITTED;
828	}
829
830	static void verity_postsuspend(struct dm_target *ti)
831	{
832	struct dm_verity *v = ti->private;
833	flush_workqueue(v->verify_wq);
834	dm_bufio_client_reset(c: v->bufio);
835	}
836
837	/*
838	* Status: V (valid) or C (corruption found)
839	*/
840	static void verity_status(struct dm_target *ti, status_type_t type,
841	unsigned int status_flags, char *result, unsigned int maxlen)
842	{
843	struct dm_verity *v = ti->private;
844	unsigned int args = 0;
845	unsigned int sz = 0;
846	unsigned int x;
847
848	switch (type) {
849	case STATUSTYPE_INFO:
850	DMEMIT("%c", v->hash_failed ? 'C' : 'V');
851	if (verity_fec_is_enabled(v))
852	DMEMIT(" %lld", atomic64_read(&v->fec->corrected));
853	else
854	DMEMIT(" -");
855	break;
856	case STATUSTYPE_TABLE:
857	DMEMIT("%u %s %s %u %u %llu %llu %s ",
858	v->version,
859	v->data_dev->name,
860	v->hash_dev->name,
861	1 << v->data_dev_block_bits,
862	1 << v->hash_dev_block_bits,
863	(unsigned long long)v->data_blocks,
864	(unsigned long long)v->hash_start,
865	v->alg_name
866	);
867	for (x = 0; x < v->digest_size; x++)
868	DMEMIT("%02x", v->root_digest[x]);
869	DMEMIT(" ");
870	if (!v->salt_size)
871	DMEMIT("-");
872	else
873	for (x = 0; x < v->salt_size; x++)
874	DMEMIT("%02x", v->salt[x]);
875	if (v->mode != DM_VERITY_MODE_EIO)
876	args++;
877	if (v->error_mode != DM_VERITY_MODE_EIO)
878	args++;
879	if (verity_fec_is_enabled(v))
880	args += DM_VERITY_OPTS_FEC;
881	if (v->zero_digest)
882	args++;
883	if (v->validated_blocks)
884	args++;
885	if (v->use_bh_wq)
886	args++;
887	if (v->signature_key_desc)
888	args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
889	if (!args)
890	return;
891	DMEMIT(" %u", args);
892	if (v->mode != DM_VERITY_MODE_EIO) {
893	DMEMIT(" ");
894	switch (v->mode) {
895	case DM_VERITY_MODE_LOGGING:
896	DMEMIT(DM_VERITY_OPT_LOGGING);
897	break;
898	case DM_VERITY_MODE_RESTART:
899	DMEMIT(DM_VERITY_OPT_RESTART);
900	break;
901	case DM_VERITY_MODE_PANIC:
902	DMEMIT(DM_VERITY_OPT_PANIC);
903	break;
904	default:
905	BUG();
906	}
907	}
908	if (v->error_mode != DM_VERITY_MODE_EIO) {
909	DMEMIT(" ");
910	switch (v->error_mode) {
911	case DM_VERITY_MODE_RESTART:
912	DMEMIT(DM_VERITY_OPT_ERROR_RESTART);
913	break;
914	case DM_VERITY_MODE_PANIC:
915	DMEMIT(DM_VERITY_OPT_ERROR_PANIC);
916	break;
917	default:
918	BUG();
919	}
920	}
921	if (v->zero_digest)
922	DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
923	if (v->validated_blocks)
924	DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
925	if (v->use_bh_wq)
926	DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
927	sz = verity_fec_status_table(v, sz, result, maxlen);
928	if (v->signature_key_desc)
929	DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY
930	" %s", v->signature_key_desc);
931	break;
932
933	case STATUSTYPE_IMA:
934	DMEMIT_TARGET_NAME_VERSION(ti->type);
935	DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V');
936	DMEMIT(",verity_version=%u", v->version);
937	DMEMIT(",data_device_name=%s", v->data_dev->name);
938	DMEMIT(",hash_device_name=%s", v->hash_dev->name);
939	DMEMIT(",verity_algorithm=%s", v->alg_name);
940
941	DMEMIT(",root_digest=");
942	for (x = 0; x < v->digest_size; x++)
943	DMEMIT("%02x", v->root_digest[x]);
944
945	DMEMIT(",salt=");
946	if (!v->salt_size)
947	DMEMIT("-");
948	else
949	for (x = 0; x < v->salt_size; x++)
950	DMEMIT("%02x", v->salt[x]);
951
952	DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n');
953	DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n');
954	if (v->signature_key_desc)
955	DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc);
956
957	if (v->mode != DM_VERITY_MODE_EIO) {
958	DMEMIT(",verity_mode=");
959	switch (v->mode) {
960	case DM_VERITY_MODE_LOGGING:
961	DMEMIT(DM_VERITY_OPT_LOGGING);
962	break;
963	case DM_VERITY_MODE_RESTART:
964	DMEMIT(DM_VERITY_OPT_RESTART);
965	break;
966	case DM_VERITY_MODE_PANIC:
967	DMEMIT(DM_VERITY_OPT_PANIC);
968	break;
969	default:
970	DMEMIT("invalid");
971	}
972	}
973	if (v->error_mode != DM_VERITY_MODE_EIO) {
974	DMEMIT(",verity_error_mode=");
975	switch (v->error_mode) {
976	case DM_VERITY_MODE_RESTART:
977	DMEMIT(DM_VERITY_OPT_ERROR_RESTART);
978	break;
979	case DM_VERITY_MODE_PANIC:
980	DMEMIT(DM_VERITY_OPT_ERROR_PANIC);
981	break;
982	default:
983	DMEMIT("invalid");
984	}
985	}
986	DMEMIT(";");
987	break;
988	}
989	}
990
991	static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev,
992	unsigned int cmd, unsigned long arg,
993	bool *forward)
994	{
995	struct dm_verity *v = ti->private;
996
997	*bdev = v->data_dev->bdev;
998
999	if (ti->len != bdev_nr_sectors(bdev: v->data_dev->bdev))
1000	return 1;
1001	return 0;
1002	}
1003
1004	static int verity_iterate_devices(struct dm_target *ti,
1005	iterate_devices_callout_fn fn, void *data)
1006	{
1007	struct dm_verity *v = ti->private;
1008
1009	return fn(ti, v->data_dev, 0, ti->len, data);
1010	}
1011
1012	static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
1013	{
1014	struct dm_verity *v = ti->private;
1015
1016	if (limits->logical_block_size < 1 << v->data_dev_block_bits)
1017	limits->logical_block_size = 1 << v->data_dev_block_bits;
1018
1019	if (limits->physical_block_size < 1 << v->data_dev_block_bits)
1020	limits->physical_block_size = 1 << v->data_dev_block_bits;
1021
1022	limits->io_min = limits->logical_block_size;
1023
1024	/*
1025	* Similar to what dm-crypt does, opt dm-verity out of support for
1026	* direct I/O that is aligned to less than the traditional direct I/O
1027	* alignment requirement of logical_block_size. This prevents dm-verity
1028	* data blocks from crossing pages, eliminating various edge cases.
1029	*/
1030	limits->dma_alignment = limits->logical_block_size - 1;
1031	}
1032
1033	#ifdef CONFIG_SECURITY
1034
1035	static int verity_init_sig(struct dm_verity *v, const void *sig,
1036	size_t sig_size)
1037	{
1038	v->sig_size = sig_size;
1039
1040	if (sig) {
1041	v->root_digest_sig = kmemdup(sig, v->sig_size, GFP_KERNEL);
1042	if (!v->root_digest_sig)
1043	return -ENOMEM;
1044	}
1045
1046	return 0;
1047	}
1048
1049	static void verity_free_sig(struct dm_verity *v)
1050	{
1051	kfree(objp: v->root_digest_sig);
1052	}
1053
1054	#else
1055
1056	static inline int verity_init_sig(struct dm_verity *v, const void *sig,
1057	size_t sig_size)
1058	{
1059	return 0;
1060	}
1061
1062	static inline void verity_free_sig(struct dm_verity *v)
1063	{
1064	}
1065
1066	#endif /* CONFIG_SECURITY */
1067
1068	static void verity_dtr(struct dm_target *ti)
1069	{
1070	struct dm_verity *v = ti->private;
1071
1072	if (v->verify_wq)
1073	destroy_workqueue(wq: v->verify_wq);
1074
1075	mempool_exit(pool: &v->recheck_pool);
1076	if (v->io)
1077	dm_io_client_destroy(client: v->io);
1078
1079	if (v->bufio)
1080	dm_bufio_client_destroy(c: v->bufio);
1081
1082	kvfree(addr: v->validated_blocks);
1083	kfree(objp: v->salt);
1084	kfree(objp: v->initial_hashstate.shash);
1085	kfree(objp: v->root_digest);
1086	kfree(objp: v->zero_digest);
1087	verity_free_sig(v);
1088
1089	crypto_free_shash(tfm: v->shash_tfm);
1090
1091	kfree(objp: v->alg_name);
1092
1093	if (v->hash_dev)
1094	dm_put_device(ti, d: v->hash_dev);
1095
1096	if (v->data_dev)
1097	dm_put_device(ti, d: v->data_dev);
1098
1099	verity_fec_dtr(v);
1100
1101	kfree(objp: v->signature_key_desc);
1102
1103	if (v->use_bh_wq)
1104	static_branch_dec(&use_bh_wq_enabled);
1105
1106	kfree(objp: v);
1107
1108	dm_audit_log_dtr(DM_MSG_PREFIX, ti, result: 1);
1109	}
1110
1111	static int verity_alloc_most_once(struct dm_verity *v)
1112	{
1113	struct dm_target *ti = v->ti;
1114
1115	if (v->validated_blocks)
1116	return 0;
1117
1118	/* the bitset can only handle INT_MAX blocks */
1119	if (v->data_blocks > INT_MAX) {
1120	ti->error = "device too large to use check_at_most_once";
1121	return -E2BIG;
1122	}
1123
1124	v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks),
1125	sizeof(unsigned long),
1126	GFP_KERNEL);
1127	if (!v->validated_blocks) {
1128	ti->error = "failed to allocate bitset for check_at_most_once";
1129	return -ENOMEM;
1130	}
1131
1132	return 0;
1133	}
1134
1135	static int verity_alloc_zero_digest(struct dm_verity *v)
1136	{
1137	int r = -ENOMEM;
1138	struct dm_verity_io *io;
1139	u8 *zero_data;
1140
1141	if (v->zero_digest)
1142	return 0;
1143
1144	v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);
1145
1146	if (!v->zero_digest)
1147	return r;
1148
1149	io = kmalloc(v->ti->per_io_data_size, GFP_KERNEL);
1150
1151	if (!io)
1152	return r; /* verity_dtr will free zero_digest */
1153
1154	zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);
1155
1156	if (!zero_data)
1157	goto out;
1158
1159	r = verity_hash(v, io, data: zero_data, len: 1 << v->data_dev_block_bits,
1160	digest: v->zero_digest);
1161
1162	out:
1163	kfree(objp: io);
1164	kfree(objp: zero_data);
1165
1166	return r;
1167	}
1168
1169	static inline bool verity_is_verity_mode(const char *arg_name)
1170	{
1171	return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_LOGGING) ||
1172	!strcasecmp(s1: arg_name, DM_VERITY_OPT_RESTART) ||
1173	!strcasecmp(s1: arg_name, DM_VERITY_OPT_PANIC));
1174	}
1175
1176	static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name)
1177	{
1178	if (v->mode)
1179	return -EINVAL;
1180
1181	if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_LOGGING))
1182	v->mode = DM_VERITY_MODE_LOGGING;
1183	else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_RESTART))
1184	v->mode = DM_VERITY_MODE_RESTART;
1185	else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_PANIC))
1186	v->mode = DM_VERITY_MODE_PANIC;
1187
1188	return 0;
1189	}
1190
1191	static inline bool verity_is_verity_error_mode(const char *arg_name)
1192	{
1193	return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_RESTART) ||
1194	!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_PANIC));
1195	}
1196
1197	static int verity_parse_verity_error_mode(struct dm_verity *v, const char *arg_name)
1198	{
1199	if (v->error_mode)
1200	return -EINVAL;
1201
1202	if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_RESTART))
1203	v->error_mode = DM_VERITY_MODE_RESTART;
1204	else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_PANIC))
1205	v->error_mode = DM_VERITY_MODE_PANIC;
1206
1207	return 0;
1208	}
1209
1210	static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
1211	struct dm_verity_sig_opts *verify_args,
1212	bool only_modifier_opts)
1213	{
1214	int r = 0;
1215	unsigned int argc;
1216	struct dm_target *ti = v->ti;
1217	const char *arg_name;
1218
1219	static const struct dm_arg _args[] = {
1220	{0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
1221	};
1222
1223	r = dm_read_arg_group(arg: _args, arg_set: as, num_args: &argc, error: &ti->error);
1224	if (r)
1225	return -EINVAL;
1226
1227	if (!argc)
1228	return 0;
1229
1230	do {
1231	arg_name = dm_shift_arg(as);
1232	argc--;
1233
1234	if (verity_is_verity_mode(arg_name)) {
1235	if (only_modifier_opts)
1236	continue;
1237	r = verity_parse_verity_mode(v, arg_name);
1238	if (r) {
1239	ti->error = "Conflicting error handling parameters";
1240	return r;
1241	}
1242	continue;
1243
1244	} else if (verity_is_verity_error_mode(arg_name)) {
1245	if (only_modifier_opts)
1246	continue;
1247	r = verity_parse_verity_error_mode(v, arg_name);
1248	if (r) {
1249	ti->error = "Conflicting error handling parameters";
1250	return r;
1251	}
1252	continue;
1253
1254	} else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_IGN_ZEROES)) {
1255	if (only_modifier_opts)
1256	continue;
1257	r = verity_alloc_zero_digest(v);
1258	if (r) {
1259	ti->error = "Cannot allocate zero digest";
1260	return r;
1261	}
1262	continue;
1263
1264	} else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) {
1265	if (only_modifier_opts)
1266	continue;
1267	r = verity_alloc_most_once(v);
1268	if (r)
1269	return r;
1270	continue;
1271
1272	} else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
1273	v->use_bh_wq = true;
1274	static_branch_inc(&use_bh_wq_enabled);
1275	continue;
1276
1277	} else if (verity_is_fec_opt_arg(arg_name)) {
1278	if (only_modifier_opts)
1279	continue;
1280	r = verity_fec_parse_opt_args(as, v, argc: &argc, arg_name);
1281	if (r)
1282	return r;
1283	continue;
1284
1285	} else if (verity_verify_is_sig_opt_arg(arg_name)) {
1286	if (only_modifier_opts)
1287	continue;
1288	r = verity_verify_sig_parse_opt_args(as, v,
1289	sig_opts: verify_args,
1290	argc: &argc, arg_name);
1291	if (r)
1292	return r;
1293	continue;
1294
1295	} else if (only_modifier_opts) {
1296	/*
1297	* Ignore unrecognized opt, could easily be an extra
1298	* argument to an option whose parsing was skipped.
1299	* Normal parsing (@only_modifier_opts=false) will
1300	* properly parse all options (and their extra args).
1301	*/
1302	continue;
1303	}
1304
1305	DMERR("Unrecognized verity feature request: %s", arg_name);
1306	ti->error = "Unrecognized verity feature request";
1307	return -EINVAL;
1308	} while (argc && !r);
1309
1310	return r;
1311	}
1312
1313	static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name)
1314	{
1315	struct dm_target *ti = v->ti;
1316	struct crypto_shash *shash;
1317
1318	v->alg_name = kstrdup(s: alg_name, GFP_KERNEL);
1319	if (!v->alg_name) {
1320	ti->error = "Cannot allocate algorithm name";
1321	return -ENOMEM;
1322	}
1323
1324	shash = crypto_alloc_shash(alg_name, type: 0, mask: 0);
1325	if (IS_ERR(ptr: shash)) {
1326	ti->error = "Cannot initialize hash function";
1327	return PTR_ERR(ptr: shash);
1328	}
1329	v->shash_tfm = shash;
1330	v->digest_size = crypto_shash_digestsize(tfm: shash);
1331	if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
1332	ti->error = "Digest size too big";
1333	return -EINVAL;
1334	}
1335	if (likely(v->version && strcmp(alg_name, "sha256") == 0)) {
1336	/*
1337	* Fast path: use the library API for reduced overhead and
1338	* interleaved hashing support.
1339	*/
1340	v->use_sha256_lib = true;
1341	if (sha256_finup_2x_is_optimized())
1342	v->use_sha256_finup_2x = true;
1343	ti->per_io_data_size =
1344	offsetofend(struct dm_verity_io, hash_ctx.sha256);
1345	} else {
1346	/* Fallback case: use the generic crypto API. */
1347	ti->per_io_data_size =
1348	offsetofend(struct dm_verity_io, hash_ctx.shash) +
1349	crypto_shash_descsize(tfm: shash);
1350	}
1351	return 0;
1352	}
1353
1354	static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg)
1355	{
1356	struct dm_target *ti = v->ti;
1357
1358	if (strcmp(arg, "-") != 0) {
1359	v->salt_size = strlen(arg) / 2;
1360	v->salt = kmalloc(v->salt_size, GFP_KERNEL);
1361	if (!v->salt) {
1362	ti->error = "Cannot allocate salt";
1363	return -ENOMEM;
1364	}
1365	if (strlen(arg) != v->salt_size * 2 ||
1366	hex2bin(dst: v->salt, src: arg, count: v->salt_size)) {
1367	ti->error = "Invalid salt";
1368	return -EINVAL;
1369	}
1370	}
1371	if (likely(v->use_sha256_lib)) {
1372	/* Implies version 1: salt at beginning */
1373	v->initial_hashstate.sha256 =
1374	kmalloc(sizeof(struct sha256_ctx), GFP_KERNEL);
1375	if (!v->initial_hashstate.sha256) {
1376	ti->error = "Cannot allocate initial hash state";
1377	return -ENOMEM;
1378	}
1379	sha256_init(ctx: v->initial_hashstate.sha256);
1380	sha256_update(ctx: v->initial_hashstate.sha256,
1381	data: v->salt, len: v->salt_size);
1382	} else if (v->version) { /* Version 1: salt at beginning */
1383	SHASH_DESC_ON_STACK(desc, v->shash_tfm);
1384	int r;
1385
1386	/*
1387	* Compute the pre-salted hash state that can be passed to
1388	* crypto_shash_import() for each block later.
1389	*/
1390	v->initial_hashstate.shash = kmalloc(
1391	crypto_shash_statesize(v->shash_tfm), GFP_KERNEL);
1392	if (!v->initial_hashstate.shash) {
1393	ti->error = "Cannot allocate initial hash state";
1394	return -ENOMEM;
1395	}
1396	desc->tfm = v->shash_tfm;
1397	r = crypto_shash_init(desc) ?:
1398	crypto_shash_update(desc, data: v->salt, len: v->salt_size) ?:
1399	crypto_shash_export(desc, out: v->initial_hashstate.shash);
1400	if (r) {
1401	ti->error = "Cannot set up initial hash state";
1402	return r;
1403	}
1404	}
1405	return 0;
1406	}
1407
1408	/*
1409	* Target parameters:
1410	* <version> The current format is version 1.
1411	* Vsn 0 is compatible with original Chromium OS releases.
1412	* <data device>
1413	* <hash device>
1414	* <data block size>
1415	* <hash block size>
1416	* <the number of data blocks>
1417	* <hash start block>
1418	* <algorithm>
1419	* <digest>
1420	* <salt> Hex string or "-" if no salt.
1421	*/
1422	static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1423	{
1424	struct dm_verity *v;
1425	struct dm_verity_sig_opts verify_args = {0};
1426	struct dm_arg_set as;
1427	unsigned int num;
1428	unsigned long long num_ll;
1429	int r;
1430	int i;
1431	sector_t hash_position;
1432	char dummy;
1433	char *root_hash_digest_to_validate;
1434
1435	v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
1436	if (!v) {
1437	ti->error = "Cannot allocate verity structure";
1438	return -ENOMEM;
1439	}
1440	ti->private = v;
1441	v->ti = ti;
1442
1443	r = verity_fec_ctr_alloc(v);
1444	if (r)
1445	goto bad;
1446
1447	if ((dm_table_get_mode(t: ti->table) & ~BLK_OPEN_READ)) {
1448	ti->error = "Device must be readonly";
1449	r = -EINVAL;
1450	goto bad;
1451	}
1452
1453	if (argc < 10) {
1454	ti->error = "Not enough arguments";
1455	r = -EINVAL;
1456	goto bad;
1457	}
1458
1459	/* Parse optional parameters that modify primary args */
1460	if (argc > 10) {
1461	as.argc = argc - 10;
1462	as.argv = argv + 10;
1463	r = verity_parse_opt_args(as: &as, v, verify_args: &verify_args, only_modifier_opts: true);
1464	if (r < 0)
1465	goto bad;
1466	}
1467
1468	if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 ||
1469	num > 1) {
1470	ti->error = "Invalid version";
1471	r = -EINVAL;
1472	goto bad;
1473	}
1474	v->version = num;
1475
1476	r = dm_get_device(ti, path: argv[1], BLK_OPEN_READ, result: &v->data_dev);
1477	if (r) {
1478	ti->error = "Data device lookup failed";
1479	goto bad;
1480	}
1481
1482	r = dm_get_device(ti, path: argv[2], BLK_OPEN_READ, result: &v->hash_dev);
1483	if (r) {
1484	ti->error = "Hash device lookup failed";
1485	goto bad;
1486	}
1487
1488	if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
1489	!num || (num & (num - 1)) ||
1490	num < bdev_logical_block_size(bdev: v->data_dev->bdev) ||
1491	num > PAGE_SIZE) {
1492	ti->error = "Invalid data device block size";
1493	r = -EINVAL;
1494	goto bad;
1495	}
1496	v->data_dev_block_bits = __ffs(num);
1497
1498	if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
1499	!num || (num & (num - 1)) ||
1500	num < bdev_logical_block_size(bdev: v->hash_dev->bdev) ||
1501	num > INT_MAX) {
1502	ti->error = "Invalid hash device block size";
1503	r = -EINVAL;
1504	goto bad;
1505	}
1506	v->hash_dev_block_bits = __ffs(num);
1507
1508	if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
1509	(sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
1510	>> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1511	ti->error = "Invalid data blocks";
1512	r = -EINVAL;
1513	goto bad;
1514	}
1515	v->data_blocks = num_ll;
1516
1517	if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
1518	ti->error = "Data device is too small";
1519	r = -EINVAL;
1520	goto bad;
1521	}
1522
1523	if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
1524	(sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))
1525	>> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1526	ti->error = "Invalid hash start";
1527	r = -EINVAL;
1528	goto bad;
1529	}
1530	v->hash_start = num_ll;
1531
1532	r = verity_setup_hash_alg(v, alg_name: argv[7]);
1533	if (r)
1534	goto bad;
1535
1536	v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
1537	if (!v->root_digest) {
1538	ti->error = "Cannot allocate root digest";
1539	r = -ENOMEM;
1540	goto bad;
1541	}
1542	if (strlen(argv[8]) != v->digest_size * 2 ||
1543	hex2bin(dst: v->root_digest, src: argv[8], count: v->digest_size)) {
1544	ti->error = "Invalid root digest";
1545	r = -EINVAL;
1546	goto bad;
1547	}
1548	root_hash_digest_to_validate = argv[8];
1549
1550	r = verity_setup_salt_and_hashstate(v, arg: argv[9]);
1551	if (r)
1552	goto bad;
1553
1554	argv += 10;
1555	argc -= 10;
1556
1557	/* Optional parameters */
1558	if (argc) {
1559	as.argc = argc;
1560	as.argv = argv;
1561	r = verity_parse_opt_args(as: &as, v, verify_args: &verify_args, only_modifier_opts: false);
1562	if (r < 0)
1563	goto bad;
1564	}
1565
1566	/* Root hash signature is an optional parameter */
1567	r = verity_verify_root_hash(data: root_hash_digest_to_validate,
1568	strlen(root_hash_digest_to_validate),
1569	sig_data: verify_args.sig,
1570	sig_len: verify_args.sig_size);
1571	if (r < 0) {
1572	ti->error = "Root hash verification failed";
1573	goto bad;
1574	}
1575
1576	r = verity_init_sig(v, sig: verify_args.sig, sig_size: verify_args.sig_size);
1577	if (r < 0) {
1578	ti->error = "Cannot allocate root digest signature";
1579	goto bad;
1580	}
1581
1582	v->hash_per_block_bits =
1583	__fls(word: (1 << v->hash_dev_block_bits) / v->digest_size);
1584
1585	v->levels = 0;
1586	if (v->data_blocks)
1587	while (v->hash_per_block_bits * v->levels < 64 &&
1588	(unsigned long long)(v->data_blocks - 1) >>
1589	(v->hash_per_block_bits * v->levels))
1590	v->levels++;
1591
1592	if (v->levels > DM_VERITY_MAX_LEVELS) {
1593	ti->error = "Too many tree levels";
1594	r = -E2BIG;
1595	goto bad;
1596	}
1597
1598	hash_position = v->hash_start;
1599	for (i = v->levels - 1; i >= 0; i--) {
1600	sector_t s;
1601
1602	v->hash_level_block[i] = hash_position;
1603	s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)
1604	>> ((i + 1) * v->hash_per_block_bits);
1605	if (hash_position + s < hash_position) {
1606	ti->error = "Hash device offset overflow";
1607	r = -E2BIG;
1608	goto bad;
1609	}
1610	hash_position += s;
1611	}
1612	v->hash_blocks = hash_position;
1613
1614	r = mempool_init_page_pool(&v->recheck_pool, 1, 0);
1615	if (unlikely(r)) {
1616	ti->error = "Cannot allocate mempool";
1617	goto bad;
1618	}
1619
1620	v->io = dm_io_client_create();
1621	if (IS_ERR(ptr: v->io)) {
1622	r = PTR_ERR(ptr: v->io);
1623	v->io = NULL;
1624	ti->error = "Cannot allocate dm io";
1625	goto bad;
1626	}
1627
1628	v->bufio = dm_bufio_client_create(bdev: v->hash_dev->bdev,
1629	block_size: 1 << v->hash_dev_block_bits, reserved_buffers: 1, aux_size: sizeof(struct buffer_aux),
1630	alloc_callback: dm_bufio_alloc_callback, NULL,
1631	flags: v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
1632	if (IS_ERR(ptr: v->bufio)) {
1633	ti->error = "Cannot initialize dm-bufio";
1634	r = PTR_ERR(ptr: v->bufio);
1635	v->bufio = NULL;
1636	goto bad;
1637	}
1638
1639	if (dm_bufio_get_device_size(c: v->bufio) < v->hash_blocks) {
1640	ti->error = "Hash device is too small";
1641	r = -E2BIG;
1642	goto bad;
1643	}
1644
1645	/*
1646	* Using WQ_HIGHPRI improves throughput and completion latency by
1647	* reducing wait times when reading from a dm-verity device.
1648	*
1649	* Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI
1650	* allows verify_wq to preempt softirq since verification in BH workqueue
1651	* will fall-back to using it for error handling (or if the bufio cache
1652	* doesn't have required hashes).
1653	*/
1654	v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1655	if (!v->verify_wq) {
1656	ti->error = "Cannot allocate workqueue";
1657	r = -ENOMEM;
1658	goto bad;
1659	}
1660
1661	r = verity_fec_ctr(v);
1662	if (r)
1663	goto bad;
1664
1665	ti->per_io_data_size = roundup(ti->per_io_data_size,
1666	__alignof__(struct dm_verity_io));
1667
1668	verity_verify_sig_opts_cleanup(sig_opts: &verify_args);
1669
1670	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 1);
1671
1672	return 0;
1673
1674	bad:
1675
1676	verity_verify_sig_opts_cleanup(sig_opts: &verify_args);
1677	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 0);
1678	verity_dtr(ti);
1679
1680	return r;
1681	}
1682
1683	/*
1684	* Get the verity mode (error behavior) of a verity target.
1685	*
1686	* Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity
1687	* target.
1688	*/
1689	int dm_verity_get_mode(struct dm_target *ti)
1690	{
1691	struct dm_verity *v = ti->private;
1692
1693	if (!dm_is_verity_target(ti))
1694	return -EINVAL;
1695
1696	return v->mode;
1697	}
1698
1699	/*
1700	* Get the root digest of a verity target.
1701	*
1702	* Returns a copy of the root digest, the caller is responsible for
1703	* freeing the memory of the digest.
1704	*/
1705	int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size)
1706	{
1707	struct dm_verity *v = ti->private;
1708
1709	if (!dm_is_verity_target(ti))
1710	return -EINVAL;
1711
1712	*root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL);
1713	if (*root_digest == NULL)
1714	return -ENOMEM;
1715
1716	*digest_size = v->digest_size;
1717
1718	return 0;
1719	}
1720
1721	#ifdef CONFIG_SECURITY
1722
1723	#ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG
1724
1725	static int verity_security_set_signature(struct block_device *bdev,
1726	struct dm_verity *v)
1727	{
1728	/*
1729	* if the dm-verity target is unsigned, v->root_digest_sig will
1730	* be NULL, and the hook call is still required to let LSMs mark
1731	* the device as unsigned. This information is crucial for LSMs to
1732	* block operations such as execution on unsigned files
1733	*/
1734	return security_bdev_setintegrity(bdev,
1735	type: LSM_INT_DMVERITY_SIG_VALID,
1736	value: v->root_digest_sig,
1737	size: v->sig_size);
1738	}
1739
1740	#else
1741
1742	static inline int verity_security_set_signature(struct block_device *bdev,
1743	struct dm_verity *v)
1744	{
1745	return 0;
1746	}
1747
1748	#endif /* CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG */
1749
1750	/*
1751	* Expose verity target's root hash and signature data to LSMs before resume.
1752	*
1753	* Returns 0 on success, or -ENOMEM if the system is out of memory.
1754	*/
1755	static int verity_preresume(struct dm_target *ti)
1756	{
1757	struct block_device *bdev;
1758	struct dm_verity_digest root_digest;
1759	struct dm_verity *v;
1760	int r;
1761
1762	v = ti->private;
1763	bdev = dm_disk(md: dm_table_get_md(t: ti->table))->part0;
1764	root_digest.digest = v->root_digest;
1765	root_digest.digest_len = v->digest_size;
1766	root_digest.alg = crypto_shash_alg_name(tfm: v->shash_tfm);
1767
1768	r = security_bdev_setintegrity(bdev, type: LSM_INT_DMVERITY_ROOTHASH, value: &root_digest,
1769	size: sizeof(root_digest));
1770	if (r)
1771	return r;
1772
1773	r = verity_security_set_signature(bdev, v);
1774	if (r)
1775	goto bad;
1776
1777	return 0;
1778
1779	bad:
1780
1781	security_bdev_setintegrity(bdev, type: LSM_INT_DMVERITY_ROOTHASH, NULL, size: 0);
1782
1783	return r;
1784	}
1785
1786	#endif /* CONFIG_SECURITY */
1787
1788	static struct target_type verity_target = {
1789	.name = "verity",
1790	/* Note: the LSMs depend on the singleton and immutable features */
1791	.features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
1792	.version = {1, 13, 0},
1793	.module = THIS_MODULE,
1794	.ctr = verity_ctr,
1795	.dtr = verity_dtr,
1796	.map = verity_map,
1797	.postsuspend = verity_postsuspend,
1798	.status = verity_status,
1799	.prepare_ioctl = verity_prepare_ioctl,
1800	.iterate_devices = verity_iterate_devices,
1801	.io_hints = verity_io_hints,
1802	#ifdef CONFIG_SECURITY
1803	.preresume = verity_preresume,
1804	#endif /* CONFIG_SECURITY */
1805	};
1806	module_dm(verity);
1807
1808	/*
1809	* Check whether a DM target is a verity target.
1810	*/
1811	bool dm_is_verity_target(struct dm_target *ti)
1812	{
1813	return ti->type == &verity_target;
1814	}
1815
1816	MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
1817	MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
1818	MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
1819	MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
1820	MODULE_LICENSE("GPL");
1821