1	/* SPDX-License-Identifier: GPL-2.0 */
2	/*
3	* fs/f2fs/f2fs.h
4	*
5	* Copyright (c) 2012 Samsung Electronics Co., Ltd.
6	* http://www.samsung.com/
7	*/
8	#ifndef _LINUX_F2FS_H
9	#define _LINUX_F2FS_H
10
11	#include <linux/uio.h>
12	#include <linux/types.h>
13	#include <linux/page-flags.h>
14	#include <linux/slab.h>
15	#include <linux/crc32.h>
16	#include <linux/magic.h>
17	#include <linux/kobject.h>
18	#include <linux/sched.h>
19	#include <linux/cred.h>
20	#include <linux/sched/mm.h>
21	#include <linux/vmalloc.h>
22	#include <linux/bio.h>
23	#include <linux/blkdev.h>
24	#include <linux/quotaops.h>
25	#include <linux/part_stat.h>
26	#include <linux/rw_hint.h>
27
28	#include <linux/fscrypt.h>
29	#include <linux/fsverity.h>
30
31	struct pagevec;
32
33	#ifdef CONFIG_F2FS_CHECK_FS
34	#define f2fs_bug_on(sbi, condition) BUG_ON(condition)
35	#else
36	#define f2fs_bug_on(sbi, condition) \
37	do { \
38	if (WARN_ON(condition)) \
39	set_sbi_flag(sbi, SBI_NEED_FSCK); \
40	} while (0)
41	#endif
42
43	enum {
44	FAULT_KMALLOC,
45	FAULT_KVMALLOC,
46	FAULT_PAGE_ALLOC,
47	FAULT_PAGE_GET,
48	FAULT_ALLOC_BIO, /* it's obsolete due to bio_alloc() will never fail */
49	FAULT_ALLOC_NID,
50	FAULT_ORPHAN,
51	FAULT_BLOCK,
52	FAULT_DIR_DEPTH,
53	FAULT_EVICT_INODE,
54	FAULT_TRUNCATE,
55	FAULT_READ_IO,
56	FAULT_CHECKPOINT,
57	FAULT_DISCARD,
58	FAULT_WRITE_IO,
59	FAULT_SLAB_ALLOC,
60	FAULT_DQUOT_INIT,
61	FAULT_LOCK_OP,
62	FAULT_BLKADDR_VALIDITY,
63	FAULT_BLKADDR_CONSISTENCE,
64	FAULT_NO_SEGMENT,
65	FAULT_INCONSISTENT_FOOTER,
66	FAULT_TIMEOUT,
67	FAULT_VMALLOC,
68	FAULT_MAX,
69	};
70
71	/* indicate which option to update */
72	enum fault_option {
73	FAULT_RATE = 1, /* only update fault rate */
74	FAULT_TYPE = 2, /* only update fault type */
75	FAULT_ALL = 4, /* reset all fault injection options/stats */
76	};
77
78	#ifdef CONFIG_F2FS_FAULT_INJECTION
79	struct f2fs_fault_info {
80	atomic_t inject_ops;
81	int inject_rate;
82	unsigned int inject_type;
83	/* Used to account total count of injection for each type */
84	unsigned int inject_count[FAULT_MAX];
85	};
86
87	extern const char *f2fs_fault_name[FAULT_MAX];
88	#define IS_FAULT_SET(fi, type) ((fi)->inject_type & BIT(type))
89
90	/* maximum retry count for injected failure */
91	#define DEFAULT_FAILURE_RETRY_COUNT 8
92	#else
93	#define DEFAULT_FAILURE_RETRY_COUNT 1
94	#endif
95
96	/*
97	* For mount options
98	*/
99	enum f2fs_mount_opt {
100	F2FS_MOUNT_DISABLE_ROLL_FORWARD,
101	F2FS_MOUNT_DISCARD,
102	F2FS_MOUNT_NOHEAP,
103	F2FS_MOUNT_XATTR_USER,
104	F2FS_MOUNT_POSIX_ACL,
105	F2FS_MOUNT_DISABLE_EXT_IDENTIFY,
106	F2FS_MOUNT_INLINE_XATTR,
107	F2FS_MOUNT_INLINE_DATA,
108	F2FS_MOUNT_INLINE_DENTRY,
109	F2FS_MOUNT_FLUSH_MERGE,
110	F2FS_MOUNT_NOBARRIER,
111	F2FS_MOUNT_FASTBOOT,
112	F2FS_MOUNT_READ_EXTENT_CACHE,
113	F2FS_MOUNT_DATA_FLUSH,
114	F2FS_MOUNT_FAULT_INJECTION,
115	F2FS_MOUNT_USRQUOTA,
116	F2FS_MOUNT_GRPQUOTA,
117	F2FS_MOUNT_PRJQUOTA,
118	F2FS_MOUNT_QUOTA,
119	F2FS_MOUNT_INLINE_XATTR_SIZE,
120	F2FS_MOUNT_RESERVE_ROOT,
121	F2FS_MOUNT_DISABLE_CHECKPOINT,
122	F2FS_MOUNT_NORECOVERY,
123	F2FS_MOUNT_ATGC,
124	F2FS_MOUNT_MERGE_CHECKPOINT,
125	F2FS_MOUNT_GC_MERGE,
126	F2FS_MOUNT_COMPRESS_CACHE,
127	F2FS_MOUNT_AGE_EXTENT_CACHE,
128	F2FS_MOUNT_NAT_BITS,
129	F2FS_MOUNT_INLINECRYPT,
130	/*
131	* Some f2fs environments expect to be able to pass the "lazytime" option
132	* string rather than using the MS_LAZYTIME flag, so this must remain.
133	*/
134	F2FS_MOUNT_LAZYTIME,
135	F2FS_MOUNT_RESERVE_NODE,
136	};
137
138	#define F2FS_OPTION(sbi) ((sbi)->mount_opt)
139	#define clear_opt(sbi, option) \
140	(F2FS_OPTION(sbi).opt &= ~BIT(F2FS_MOUNT_##option))
141	#define set_opt(sbi, option) \
142	(F2FS_OPTION(sbi).opt |= BIT(F2FS_MOUNT_##option))
143	#define test_opt(sbi, option) \
144	(F2FS_OPTION(sbi).opt & BIT(F2FS_MOUNT_##option))
145
146	#define ver_after(a, b) (typecheck(unsigned long long, a) && \
147	typecheck(unsigned long long, b) && \
148	((long long)((a) - (b)) > 0))
149
150	typedef u32 block_t; /*
151	* should not change u32, since it is the on-disk block
152	* address format, __le32.
153	*/
154	typedef u32 nid_t;
155
156	#define COMPRESS_EXT_NUM 16
157
158	enum blkzone_allocation_policy {
159	BLKZONE_ALLOC_PRIOR_SEQ, /* Prioritize writing to sequential zones */
160	BLKZONE_ALLOC_ONLY_SEQ, /* Only allow writing to sequential zones */
161	BLKZONE_ALLOC_PRIOR_CONV, /* Prioritize writing to conventional zones */
162	};
163
164	enum bggc_io_aware_policy {
165	AWARE_ALL_IO, /* skip background GC if there is any kind of pending IO */
166	AWARE_READ_IO, /* skip background GC if there is pending read IO */
167	AWARE_NONE, /* don't aware IO for background GC */
168	};
169
170	enum device_allocation_policy {
171	ALLOCATE_FORWARD_NOHINT,
172	ALLOCATE_FORWARD_WITHIN_HINT,
173	ALLOCATE_FORWARD_FROM_HINT,
174	};
175
176	/*
177	* An implementation of an rwsem that is explicitly unfair to readers. This
178	* prevents priority inversion when a low-priority reader acquires the read lock
179	* while sleeping on the write lock but the write lock is needed by
180	* higher-priority clients.
181	*/
182
183	struct f2fs_rwsem {
184	struct rw_semaphore internal_rwsem;
185	#ifdef CONFIG_F2FS_UNFAIR_RWSEM
186	wait_queue_head_t read_waiters;
187	#endif
188	};
189
190	struct f2fs_mount_info {
191	unsigned long long opt;
192	block_t root_reserved_blocks; /* root reserved blocks */
193	block_t root_reserved_nodes; /* root reserved nodes */
194	kuid_t s_resuid; /* reserved blocks for uid */
195	kgid_t s_resgid; /* reserved blocks for gid */
196	int active_logs; /* # of active logs */
197	int inline_xattr_size; /* inline xattr size */
198	#ifdef CONFIG_F2FS_FAULT_INJECTION
199	struct f2fs_fault_info fault_info; /* For fault injection */
200	#endif
201	#ifdef CONFIG_QUOTA
202	/* Names of quota files with journalled quota */
203	char *s_qf_names[MAXQUOTAS];
204	int s_jquota_fmt; /* Format of quota to use */
205	#endif
206	/* For which write hints are passed down to block layer */
207	int alloc_mode; /* segment allocation policy */
208	int fsync_mode; /* fsync policy */
209	int fs_mode; /* fs mode: LFS or ADAPTIVE */
210	int bggc_mode; /* bggc mode: off, on or sync */
211	int memory_mode; /* memory mode */
212	int errors; /* errors parameter */
213	int discard_unit; /*
214	* discard command's offset/size should
215	* be aligned to this unit: block,
216	* segment or section
217	*/
218	struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
219	block_t unusable_cap_perc; /* percentage for cap */
220	block_t unusable_cap; /* Amount of space allowed to be
221	* unusable when disabling checkpoint
222	*/
223
224	/* For compression */
225	unsigned char compress_algorithm; /* algorithm type */
226	unsigned char compress_log_size; /* cluster log size */
227	unsigned char compress_level; /* compress level */
228	bool compress_chksum; /* compressed data chksum */
229	unsigned char compress_ext_cnt; /* extension count */
230	unsigned char nocompress_ext_cnt; /* nocompress extension count */
231	int compress_mode; /* compression mode */
232	unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */
233	unsigned char noextensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */
234	unsigned int lookup_mode;
235	};
236
237	#define F2FS_FEATURE_ENCRYPT 0x00000001
238	#define F2FS_FEATURE_BLKZONED 0x00000002
239	#define F2FS_FEATURE_ATOMIC_WRITE 0x00000004
240	#define F2FS_FEATURE_EXTRA_ATTR 0x00000008
241	#define F2FS_FEATURE_PRJQUOTA 0x00000010
242	#define F2FS_FEATURE_INODE_CHKSUM 0x00000020
243	#define F2FS_FEATURE_FLEXIBLE_INLINE_XATTR 0x00000040
244	#define F2FS_FEATURE_QUOTA_INO 0x00000080
245	#define F2FS_FEATURE_INODE_CRTIME 0x00000100
246	#define F2FS_FEATURE_LOST_FOUND 0x00000200
247	#define F2FS_FEATURE_VERITY 0x00000400
248	#define F2FS_FEATURE_SB_CHKSUM 0x00000800
249	#define F2FS_FEATURE_CASEFOLD 0x00001000
250	#define F2FS_FEATURE_COMPRESSION 0x00002000
251	#define F2FS_FEATURE_RO 0x00004000
252	#define F2FS_FEATURE_DEVICE_ALIAS 0x00008000
253	#define F2FS_FEATURE_PACKED_SSA 0x00010000
254
255	#define __F2FS_HAS_FEATURE(raw_super, mask) \
256	((raw_super->feature & cpu_to_le32(mask)) != 0)
257	#define F2FS_HAS_FEATURE(sbi, mask) __F2FS_HAS_FEATURE(sbi->raw_super, mask)
258
259	/*
260	* Default values for user and/or group using reserved blocks
261	*/
262	#define F2FS_DEF_RESUID 0
263	#define F2FS_DEF_RESGID 0
264
265	/*
266	* For checkpoint manager
267	*/
268	enum {
269	NAT_BITMAP,
270	SIT_BITMAP
271	};
272
273	#define CP_UMOUNT 0x00000001
274	#define CP_FASTBOOT 0x00000002
275	#define CP_SYNC 0x00000004
276	#define CP_RECOVERY 0x00000008
277	#define CP_DISCARD 0x00000010
278	#define CP_TRIMMED 0x00000020
279	#define CP_PAUSE 0x00000040
280	#define CP_RESIZE 0x00000080
281
282	#define DEF_MAX_DISCARD_REQUEST 8 /* issue 8 discards per round */
283	#define DEF_MIN_DISCARD_ISSUE_TIME 50 /* 50 ms, if exists */
284	#define DEF_MID_DISCARD_ISSUE_TIME 500 /* 500 ms, if device busy */
285	#define DEF_MAX_DISCARD_ISSUE_TIME 60000 /* 60 s, if no candidates */
286	#define DEF_DISCARD_URGENT_UTIL 80 /* do more discard over 80% */
287	#define DEF_CP_INTERVAL 60 /* 60 secs */
288	#define DEF_IDLE_INTERVAL 5 /* 5 secs */
289	#define DEF_DISABLE_INTERVAL 5 /* 5 secs */
290	#define DEF_ENABLE_INTERVAL 5 /* 5 secs */
291	#define DEF_DISABLE_QUICK_INTERVAL 1 /* 1 secs */
292	#define DEF_UMOUNT_DISCARD_TIMEOUT 5 /* 5 secs */
293
294	enum cp_time {
295	CP_TIME_START, /* begin */
296	CP_TIME_LOCK, /* after cp_global_sem */
297	CP_TIME_OP_LOCK, /* after block_operation */
298	CP_TIME_FLUSH_META, /* after flush sit/nat */
299	CP_TIME_SYNC_META, /* after sync_meta_pages */
300	CP_TIME_SYNC_CP_META, /* after sync cp meta pages */
301	CP_TIME_WAIT_DIRTY_META,/* after wait on dirty meta */
302	CP_TIME_WAIT_CP_DATA, /* after wait on cp data */
303	CP_TIME_FLUSH_DEVICE, /* after flush device cache */
304	CP_TIME_WAIT_LAST_CP, /* after wait on last cp pack */
305	CP_TIME_END, /* after unblock_operation */
306	CP_TIME_MAX,
307	};
308
309	/* time cost stats of checkpoint */
310	struct cp_stats {
311	ktime_t times[CP_TIME_MAX];
312	};
313
314	struct cp_control {
315	int reason;
316	__u64 trim_start;
317	__u64 trim_end;
318	__u64 trim_minlen;
319	struct cp_stats stats;
320	};
321
322	enum f2fs_cp_phase {
323	CP_PHASE_START_BLOCK_OPS,
324	CP_PHASE_FINISH_BLOCK_OPS,
325	CP_PHASE_FINISH_CHECKPOINT,
326	};
327
328	/*
329	* indicate meta/data type
330	*/
331	enum {
332	META_CP,
333	META_NAT,
334	META_SIT,
335	META_SSA,
336	META_MAX,
337	META_POR,
338	DATA_GENERIC, /* check range only */
339	DATA_GENERIC_ENHANCE, /* strong check on range and segment bitmap */
340	DATA_GENERIC_ENHANCE_READ, /*
341	* strong check on range and segment
342	* bitmap but no warning due to race
343	* condition of read on truncated area
344	* by extent_cache
345	*/
346	DATA_GENERIC_ENHANCE_UPDATE, /*
347	* strong check on range and segment
348	* bitmap for update case
349	*/
350	META_GENERIC,
351	};
352
353	/* for the list of ino */
354	enum {
355	ORPHAN_INO, /* for orphan ino list */
356	APPEND_INO, /* for append ino list */
357	UPDATE_INO, /* for update ino list */
358	TRANS_DIR_INO, /* for transactions dir ino list */
359	XATTR_DIR_INO, /* for xattr updated dir ino list */
360	FLUSH_INO, /* for multiple device flushing */
361	MAX_INO_ENTRY, /* max. list */
362	};
363
364	struct ino_entry {
365	struct list_head list; /* list head */
366	nid_t ino; /* inode number */
367	unsigned int dirty_device; /* dirty device bitmap */
368	};
369
370	/* for the list of inodes to be GCed */
371	struct inode_entry {
372	struct list_head list; /* list head */
373	struct inode *inode; /* vfs inode pointer */
374	};
375
376	struct fsync_node_entry {
377	struct list_head list; /* list head */
378	struct folio *folio; /* warm node folio pointer */
379	unsigned int seq_id; /* sequence id */
380	};
381
382	struct ckpt_req {
383	struct completion wait; /* completion for checkpoint done */
384	struct llist_node llnode; /* llist_node to be linked in wait queue */
385	int ret; /* return code of checkpoint */
386	union {
387	ktime_t queue_time; /* request queued time */
388	ktime_t delta_time; /* time in queue */
389	};
390	};
391
392	struct ckpt_req_control {
393	struct task_struct *f2fs_issue_ckpt; /* checkpoint task */
394	int ckpt_thread_ioprio; /* checkpoint merge thread ioprio */
395	wait_queue_head_t ckpt_wait_queue; /* waiting queue for wake-up */
396	atomic_t issued_ckpt; /* # of actually issued ckpts */
397	atomic_t total_ckpt; /* # of total ckpts */
398	atomic_t queued_ckpt; /* # of queued ckpts */
399	struct llist_head issue_list; /* list for command issue */
400	spinlock_t stat_lock; /* lock for below checkpoint time stats */
401	unsigned int cur_time; /* cur wait time in msec for currently issued checkpoint */
402	unsigned int peak_time; /* peak wait time in msec until now */
403	};
404
405	/* a time threshold that checkpoint was blocked for, unit: ms */
406	#define CP_LONG_LATENCY_THRESHOLD 5000
407
408	/* for the bitmap indicate blocks to be discarded */
409	struct discard_entry {
410	struct list_head list; /* list head */
411	block_t start_blkaddr; /* start blockaddr of current segment */
412	unsigned char discard_map[SIT_VBLOCK_MAP_SIZE]; /* segment discard bitmap */
413	};
414
415	/* minimum discard granularity, unit: block count */
416	#define MIN_DISCARD_GRANULARITY 1
417	/* default discard granularity of inner discard thread, unit: block count */
418	#define DEFAULT_DISCARD_GRANULARITY 16
419	/* default maximum discard granularity of ordered discard, unit: block count */
420	#define DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY 16
421	/* default interval of periodical discard submission */
422	#define DEFAULT_DISCARD_INTERVAL (msecs_to_jiffies(20))
423
424	/* max discard pend list number */
425	#define MAX_PLIST_NUM 512
426	#define plist_idx(blk_num) ((blk_num) >= MAX_PLIST_NUM ? \
427	(MAX_PLIST_NUM - 1) : ((blk_num) - 1))
428
429	enum {
430	D_PREP, /* initial */
431	D_PARTIAL, /* partially submitted */
432	D_SUBMIT, /* all submitted */
433	D_DONE, /* finished */
434	};
435
436	struct discard_info {
437	block_t lstart; /* logical start address */
438	block_t len; /* length */
439	block_t start; /* actual start address in dev */
440	};
441
442	struct discard_cmd {
443	struct rb_node rb_node; /* rb node located in rb-tree */
444	struct discard_info di; /* discard info */
445	struct list_head list; /* command list */
446	struct completion wait; /* completion */
447	struct block_device *bdev; /* bdev */
448	unsigned short ref; /* reference count */
449	unsigned char state; /* state */
450	unsigned char queued; /* queued discard */
451	int error; /* bio error */
452	spinlock_t lock; /* for state/bio_ref updating */
453	unsigned short bio_ref; /* bio reference count */
454	};
455
456	enum {
457	DPOLICY_BG,
458	DPOLICY_FORCE,
459	DPOLICY_FSTRIM,
460	DPOLICY_UMOUNT,
461	MAX_DPOLICY,
462	};
463
464	enum {
465	DPOLICY_IO_AWARE_DISABLE, /* force to not be aware of IO */
466	DPOLICY_IO_AWARE_ENABLE, /* force to be aware of IO */
467	DPOLICY_IO_AWARE_MAX,
468	};
469
470	struct discard_policy {
471	int type; /* type of discard */
472	unsigned int min_interval; /* used for candidates exist */
473	unsigned int mid_interval; /* used for device busy */
474	unsigned int max_interval; /* used for candidates not exist */
475	unsigned int max_requests; /* # of discards issued per round */
476	unsigned int io_aware_gran; /* minimum granularity discard not be aware of I/O */
477	bool io_aware; /* issue discard in idle time */
478	bool sync; /* submit discard with REQ_SYNC flag */
479	bool ordered; /* issue discard by lba order */
480	bool timeout; /* discard timeout for put_super */
481	unsigned int granularity; /* discard granularity */
482	};
483
484	struct discard_cmd_control {
485	struct task_struct *f2fs_issue_discard; /* discard thread */
486	struct list_head entry_list; /* 4KB discard entry list */
487	struct list_head pend_list[MAX_PLIST_NUM];/* store pending entries */
488	struct list_head wait_list; /* store on-flushing entries */
489	struct list_head fstrim_list; /* in-flight discard from fstrim */
490	wait_queue_head_t discard_wait_queue; /* waiting queue for wake-up */
491	struct mutex cmd_lock;
492	unsigned int nr_discards; /* # of discards in the list */
493	unsigned int max_discards; /* max. discards to be issued */
494	unsigned int max_discard_request; /* max. discard request per round */
495	unsigned int min_discard_issue_time; /* min. interval between discard issue */
496	unsigned int mid_discard_issue_time; /* mid. interval between discard issue */
497	unsigned int max_discard_issue_time; /* max. interval between discard issue */
498	unsigned int discard_io_aware_gran; /* minimum discard granularity not be aware of I/O */
499	unsigned int discard_urgent_util; /* utilization which issue discard proactively */
500	unsigned int discard_granularity; /* discard granularity */
501	unsigned int max_ordered_discard; /* maximum discard granularity issued by lba order */
502	unsigned int discard_io_aware; /* io_aware policy */
503	unsigned int undiscard_blks; /* # of undiscard blocks */
504	unsigned int next_pos; /* next discard position */
505	atomic_t issued_discard; /* # of issued discard */
506	atomic_t queued_discard; /* # of queued discard */
507	atomic_t discard_cmd_cnt; /* # of cached cmd count */
508	struct rb_root_cached root; /* root of discard rb-tree */
509	bool rbtree_check; /* config for consistence check */
510	bool discard_wake; /* to wake up discard thread */
511	};
512
513	/* for the list of fsync inodes, used only during recovery */
514	struct fsync_inode_entry {
515	struct list_head list; /* list head */
516	struct inode *inode; /* vfs inode pointer */
517	block_t blkaddr; /* block address locating the last fsync */
518	block_t last_dentry; /* block address locating the last dentry */
519	};
520
521	#define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats))
522	#define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits))
523
524	#define nat_in_journal(jnl, i) ((jnl)->nat_j.entries[i].ne)
525	#define nid_in_journal(jnl, i) ((jnl)->nat_j.entries[i].nid)
526	#define sit_in_journal(jnl, i) ((jnl)->sit_j.entries[i].se)
527	#define segno_in_journal(jnl, i) ((jnl)->sit_j.entries[i].segno)
528
529	#define MAX_NAT_JENTRIES(jnl) (NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl))
530	#define MAX_SIT_JENTRIES(jnl) (SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl))
531
532	static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i)
533	{
534	int before = nats_in_cursum(journal);
535
536	journal->n_nats = cpu_to_le16(before + i);
537	return before;
538	}
539
540	static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i)
541	{
542	int before = sits_in_cursum(journal);
543
544	journal->n_sits = cpu_to_le16(before + i);
545	return before;
546	}
547
548	static inline bool __has_cursum_space(struct f2fs_journal *journal,
549	int size, int type)
550	{
551	if (type == NAT_JOURNAL)
552	return size <= MAX_NAT_JENTRIES(journal);
553	return size <= MAX_SIT_JENTRIES(journal);
554	}
555
556	/* for inline stuff */
557	#define DEF_INLINE_RESERVED_SIZE 1
558	static inline int get_extra_isize(struct inode *inode);
559	static inline int get_inline_xattr_addrs(struct inode *inode);
560	#define MAX_INLINE_DATA(inode) (sizeof(__le32) * \
561	(CUR_ADDRS_PER_INODE(inode) - \
562	get_inline_xattr_addrs(inode) - \
563	DEF_INLINE_RESERVED_SIZE))
564
565	/* for inline dir */
566	#define NR_INLINE_DENTRY(inode) (MAX_INLINE_DATA(inode) * BITS_PER_BYTE / \
567	((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
568	BITS_PER_BYTE + 1))
569	#define INLINE_DENTRY_BITMAP_SIZE(inode) \
570	DIV_ROUND_UP(NR_INLINE_DENTRY(inode), BITS_PER_BYTE)
571	#define INLINE_RESERVED_SIZE(inode) (MAX_INLINE_DATA(inode) - \
572	((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
573	NR_INLINE_DENTRY(inode) + \
574	INLINE_DENTRY_BITMAP_SIZE(inode)))
575
576	/*
577	* For INODE and NODE manager
578	*/
579	/* for directory operations */
580
581	struct f2fs_filename {
582	/*
583	* The filename the user specified. This is NULL for some
584	* filesystem-internal operations, e.g. converting an inline directory
585	* to a non-inline one, or roll-forward recovering an encrypted dentry.
586	*/
587	const struct qstr *usr_fname;
588
589	/*
590	* The on-disk filename. For encrypted directories, this is encrypted.
591	* This may be NULL for lookups in an encrypted dir without the key.
592	*/
593	struct fscrypt_str disk_name;
594
595	/* The dirhash of this filename */
596	f2fs_hash_t hash;
597
598	#ifdef CONFIG_FS_ENCRYPTION
599	/*
600	* For lookups in encrypted directories: either the buffer backing
601	* disk_name, or a buffer that holds the decoded no-key name.
602	*/
603	struct fscrypt_str crypto_buf;
604	#endif
605	#if IS_ENABLED(CONFIG_UNICODE)
606	/*
607	* For casefolded directories: the casefolded name, but it's left NULL
608	* if the original name is not valid Unicode, if the original name is
609	* "." or "..", if the directory is both casefolded and encrypted and
610	* its encryption key is unavailable, or if the filesystem is doing an
611	* internal operation where usr_fname is also NULL. In all these cases
612	* we fall back to treating the name as an opaque byte sequence.
613	*/
614	struct qstr cf_name;
615	#endif
616	};
617
618	struct f2fs_dentry_ptr {
619	struct inode *inode;
620	void *bitmap;
621	struct f2fs_dir_entry *dentry;
622	__u8 (*filename)[F2FS_SLOT_LEN];
623	int max;
624	int nr_bitmap;
625	};
626
627	static inline void make_dentry_ptr_block(struct inode *inode,
628	struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t)
629	{
630	d->inode = inode;
631	d->max = NR_DENTRY_IN_BLOCK;
632	d->nr_bitmap = SIZE_OF_DENTRY_BITMAP;
633	d->bitmap = t->dentry_bitmap;
634	d->dentry = t->dentry;
635	d->filename = t->filename;
636	}
637
638	static inline void make_dentry_ptr_inline(struct inode *inode,
639	struct f2fs_dentry_ptr *d, void *t)
640	{
641	int entry_cnt = NR_INLINE_DENTRY(inode);
642	int bitmap_size = INLINE_DENTRY_BITMAP_SIZE(inode);
643	int reserved_size = INLINE_RESERVED_SIZE(inode);
644
645	d->inode = inode;
646	d->max = entry_cnt;
647	d->nr_bitmap = bitmap_size;
648	d->bitmap = t;
649	d->dentry = t + bitmap_size + reserved_size;
650	d->filename = t + bitmap_size + reserved_size +
651	SIZE_OF_DIR_ENTRY * entry_cnt;
652	}
653
654	/*
655	* XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1
656	* as its node offset to distinguish from index node blocks.
657	* But some bits are used to mark the node block.
658	*/
659	#define XATTR_NODE_OFFSET ((((unsigned int)-1) << OFFSET_BIT_SHIFT) \
660	>> OFFSET_BIT_SHIFT)
661	enum {
662	ALLOC_NODE, /* allocate a new node page if needed */
663	LOOKUP_NODE, /* look up a node without readahead */
664	LOOKUP_NODE_RA, /*
665	* look up a node with readahead called
666	* by get_data_block.
667	*/
668	};
669
670	#define DEFAULT_RETRY_IO_COUNT 8 /* maximum retry read IO or flush count */
671
672	/* IO/non-IO congestion wait timeout value, default: 1ms */
673	#define DEFAULT_SCHEDULE_TIMEOUT (msecs_to_jiffies(1))
674
675	/* timeout value injected, default: 1000ms */
676	#define DEFAULT_FAULT_TIMEOUT (msecs_to_jiffies(1000))
677
678	/* maximum retry quota flush count */
679	#define DEFAULT_RETRY_QUOTA_FLUSH_COUNT 8
680
681	/* maximum retry of EIO'ed page */
682	#define MAX_RETRY_PAGE_EIO 100
683
684	#define F2FS_LINK_MAX 0xffffffff /* maximum link count per file */
685
686	#define MAX_DIR_RA_PAGES 4 /* maximum ra pages of dir */
687
688	/* dirty segments threshold for triggering CP */
689	#define DEFAULT_DIRTY_THRESHOLD 4
690
691	#define RECOVERY_MAX_RA_BLOCKS BIO_MAX_VECS
692	#define RECOVERY_MIN_RA_BLOCKS 1
693
694	#define F2FS_ONSTACK_PAGES 16 /* nr of onstack pages */
695
696	/* for in-memory extent cache entry */
697	#define F2FS_MIN_EXTENT_LEN 64 /* minimum extent length */
698
699	/* number of extent info in extent cache we try to shrink */
700	#define READ_EXTENT_CACHE_SHRINK_NUMBER 128
701
702	/* number of age extent info in extent cache we try to shrink */
703	#define AGE_EXTENT_CACHE_SHRINK_NUMBER 128
704	#define LAST_AGE_WEIGHT 30
705	#define SAME_AGE_REGION 1024
706
707	/*
708	* Define data block with age less than 1GB as hot data
709	* define data block with age less than 10GB but more than 1GB as warm data
710	*/
711	#define DEF_HOT_DATA_AGE_THRESHOLD 262144
712	#define DEF_WARM_DATA_AGE_THRESHOLD 2621440
713
714	/* default max read extent count per inode */
715	#define DEF_MAX_READ_EXTENT_COUNT 10240
716
717	/* extent cache type */
718	enum extent_type {
719	EX_READ,
720	EX_BLOCK_AGE,
721	NR_EXTENT_CACHES,
722	};
723
724	/*
725	* Reserved value to mark invalid age extents, hence valid block range
726	* from 0 to ULLONG_MAX-1
727	*/
728	#define F2FS_EXTENT_AGE_INVALID ULLONG_MAX
729
730	struct extent_info {
731	unsigned int fofs; /* start offset in a file */
732	unsigned int len; /* length of the extent */
733	union {
734	/* read extent_cache */
735	struct {
736	/* start block address of the extent */
737	block_t blk;
738	#ifdef CONFIG_F2FS_FS_COMPRESSION
739	/* physical extent length of compressed blocks */
740	unsigned int c_len;
741	#endif
742	};
743	/* block age extent_cache */
744	struct {
745	/* block age of the extent */
746	unsigned long long age;
747	/* last total blocks allocated */
748	unsigned long long last_blocks;
749	};
750	};
751	};
752
753	struct extent_node {
754	struct rb_node rb_node; /* rb node located in rb-tree */
755	struct extent_info ei; /* extent info */
756	struct list_head list; /* node in global extent list of sbi */
757	struct extent_tree *et; /* extent tree pointer */
758	};
759
760	struct extent_tree {
761	nid_t ino; /* inode number */
762	enum extent_type type; /* keep the extent tree type */
763	struct rb_root_cached root; /* root of extent info rb-tree */
764	struct extent_node *cached_en; /* recently accessed extent node */
765	struct list_head list; /* to be used by sbi->zombie_list */
766	rwlock_t lock; /* protect extent info rb-tree */
767	atomic_t node_cnt; /* # of extent node in rb-tree*/
768	bool largest_updated; /* largest extent updated */
769	struct extent_info largest; /* largest cached extent for EX_READ */
770	};
771
772	struct extent_tree_info {
773	struct radix_tree_root extent_tree_root;/* cache extent cache entries */
774	struct mutex extent_tree_lock; /* locking extent radix tree */
775	struct list_head extent_list; /* lru list for shrinker */
776	spinlock_t extent_lock; /* locking extent lru list */
777	atomic_t total_ext_tree; /* extent tree count */
778	struct list_head zombie_list; /* extent zombie tree list */
779	atomic_t total_zombie_tree; /* extent zombie tree count */
780	atomic_t total_ext_node; /* extent info count */
781	};
782
783	/*
784	* State of block returned by f2fs_map_blocks.
785	*/
786	#define F2FS_MAP_NEW (1U << 0)
787	#define F2FS_MAP_MAPPED (1U << 1)
788	#define F2FS_MAP_DELALLOC (1U << 2)
789	#define F2FS_MAP_FLAGS (F2FS_MAP_NEW | F2FS_MAP_MAPPED |\
790	F2FS_MAP_DELALLOC)
791
792	struct f2fs_map_blocks {
793	struct block_device *m_bdev; /* for multi-device dio */
794	block_t m_pblk;
795	block_t m_lblk;
796	unsigned int m_len;
797	unsigned int m_flags;
798	unsigned long m_last_pblk; /* last allocated block, only used for DIO in LFS mode */
799	pgoff_t *m_next_pgofs; /* point next possible non-hole pgofs */
800	pgoff_t *m_next_extent; /* point to next possible extent */
801	int m_seg_type;
802	bool m_may_create; /* indicate it is from write path */
803	bool m_multidev_dio; /* indicate it allows multi-device dio */
804	};
805
806	/* for flag in get_data_block */
807	enum {
808	F2FS_GET_BLOCK_DEFAULT,
809	F2FS_GET_BLOCK_FIEMAP,
810	F2FS_GET_BLOCK_BMAP,
811	F2FS_GET_BLOCK_DIO,
812	F2FS_GET_BLOCK_PRE_DIO,
813	F2FS_GET_BLOCK_PRE_AIO,
814	F2FS_GET_BLOCK_PRECACHE,
815	};
816
817	/*
818	* i_advise uses FADVISE_XXX_BIT. We can add additional hints later.
819	*/
820	#define FADVISE_COLD_BIT 0x01
821	#define FADVISE_LOST_PINO_BIT 0x02
822	#define FADVISE_ENCRYPT_BIT 0x04
823	#define FADVISE_ENC_NAME_BIT 0x08
824	#define FADVISE_KEEP_SIZE_BIT 0x10
825	#define FADVISE_HOT_BIT 0x20
826	#define FADVISE_VERITY_BIT 0x40
827	#define FADVISE_TRUNC_BIT 0x80
828
829	#define FADVISE_MODIFIABLE_BITS (FADVISE_COLD_BIT | FADVISE_HOT_BIT)
830
831	#define file_is_cold(inode) is_file(inode, FADVISE_COLD_BIT)
832	#define file_set_cold(inode) set_file(inode, FADVISE_COLD_BIT)
833	#define file_clear_cold(inode) clear_file(inode, FADVISE_COLD_BIT)
834
835	#define file_wrong_pino(inode) is_file(inode, FADVISE_LOST_PINO_BIT)
836	#define file_lost_pino(inode) set_file(inode, FADVISE_LOST_PINO_BIT)
837	#define file_got_pino(inode) clear_file(inode, FADVISE_LOST_PINO_BIT)
838
839	#define file_is_encrypt(inode) is_file(inode, FADVISE_ENCRYPT_BIT)
840	#define file_set_encrypt(inode) set_file(inode, FADVISE_ENCRYPT_BIT)
841
842	#define file_enc_name(inode) is_file(inode, FADVISE_ENC_NAME_BIT)
843	#define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
844
845	#define file_keep_isize(inode) is_file(inode, FADVISE_KEEP_SIZE_BIT)
846	#define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
847
848	#define file_is_hot(inode) is_file(inode, FADVISE_HOT_BIT)
849	#define file_set_hot(inode) set_file(inode, FADVISE_HOT_BIT)
850	#define file_clear_hot(inode) clear_file(inode, FADVISE_HOT_BIT)
851
852	#define file_is_verity(inode) is_file(inode, FADVISE_VERITY_BIT)
853	#define file_set_verity(inode) set_file(inode, FADVISE_VERITY_BIT)
854
855	#define file_should_truncate(inode) is_file(inode, FADVISE_TRUNC_BIT)
856	#define file_need_truncate(inode) set_file(inode, FADVISE_TRUNC_BIT)
857	#define file_dont_truncate(inode) clear_file(inode, FADVISE_TRUNC_BIT)
858
859	#define DEF_DIR_LEVEL 0
860
861	/* used for f2fs_inode_info->flags */
862	enum {
863	FI_NEW_INODE, /* indicate newly allocated inode */
864	FI_DIRTY_INODE, /* indicate inode is dirty or not */
865	FI_AUTO_RECOVER, /* indicate inode is recoverable */
866	FI_DIRTY_DIR, /* indicate directory has dirty pages */
867	FI_INC_LINK, /* need to increment i_nlink */
868	FI_ACL_MODE, /* indicate acl mode */
869	FI_NO_ALLOC, /* should not allocate any blocks */
870	FI_FREE_NID, /* free allocated nide */
871	FI_NO_EXTENT, /* not to use the extent cache */
872	FI_INLINE_XATTR, /* used for inline xattr */
873	FI_INLINE_DATA, /* used for inline data*/
874	FI_INLINE_DENTRY, /* used for inline dentry */
875	FI_APPEND_WRITE, /* inode has appended data */
876	FI_UPDATE_WRITE, /* inode has in-place-update data */
877	FI_NEED_IPU, /* used for ipu per file */
878	FI_ATOMIC_FILE, /* indicate atomic file */
879	FI_DATA_EXIST, /* indicate data exists */
880	FI_SKIP_WRITES, /* should skip data page writeback */
881	FI_OPU_WRITE, /* used for opu per file */
882	FI_DIRTY_FILE, /* indicate regular/symlink has dirty pages */
883	FI_PREALLOCATED_ALL, /* all blocks for write were preallocated */
884	FI_HOT_DATA, /* indicate file is hot */
885	FI_EXTRA_ATTR, /* indicate file has extra attribute */
886	FI_PROJ_INHERIT, /* indicate file inherits projectid */
887	FI_PIN_FILE, /* indicate file should not be gced */
888	FI_VERITY_IN_PROGRESS, /* building fs-verity Merkle tree */
889	FI_COMPRESSED_FILE, /* indicate file's data can be compressed */
890	FI_COMPRESS_CORRUPT, /* indicate compressed cluster is corrupted */
891	FI_MMAP_FILE, /* indicate file was mmapped */
892	FI_ENABLE_COMPRESS, /* enable compression in "user" compression mode */
893	FI_COMPRESS_RELEASED, /* compressed blocks were released */
894	FI_ALIGNED_WRITE, /* enable aligned write */
895	FI_COW_FILE, /* indicate COW file */
896	FI_ATOMIC_COMMITTED, /* indicate atomic commit completed except disk sync */
897	FI_ATOMIC_DIRTIED, /* indicate atomic file is dirtied */
898	FI_ATOMIC_REPLACE, /* indicate atomic replace */
899	FI_OPENED_FILE, /* indicate file has been opened */
900	FI_DONATE_FINISHED, /* indicate page donation of file has been finished */
901	FI_MAX, /* max flag, never be used */
902	};
903
904	struct f2fs_inode_info {
905	struct inode vfs_inode; /* serve a vfs inode */
906	unsigned long i_flags; /* keep an inode flags for ioctl */
907	unsigned char i_advise; /* use to give file attribute hints */
908	unsigned char i_dir_level; /* use for dentry level for large dir */
909	union {
910	unsigned int i_current_depth; /* only for directory depth */
911	unsigned short i_gc_failures; /* for gc failure statistic */
912	};
913	unsigned int i_pino; /* parent inode number */
914	umode_t i_acl_mode; /* keep file acl mode temporarily */
915
916	/* Use below internally in f2fs*/
917	unsigned long flags[BITS_TO_LONGS(FI_MAX)]; /* use to pass per-file flags */
918	unsigned int ioprio_hint; /* hint for IO priority */
919	struct f2fs_rwsem i_sem; /* protect fi info */
920	atomic_t dirty_pages; /* # of dirty pages */
921	f2fs_hash_t chash; /* hash value of given file name */
922	unsigned int clevel; /* maximum level of given file name */
923	struct task_struct *task; /* lookup and create consistency */
924	struct task_struct *cp_task; /* separate cp/wb IO stats*/
925	struct task_struct *wb_task; /* indicate inode is in context of writeback */
926	nid_t i_xattr_nid; /* node id that contains xattrs */
927	loff_t last_disk_size; /* lastly written file size */
928	spinlock_t i_size_lock; /* protect last_disk_size */
929
930	#ifdef CONFIG_QUOTA
931	struct dquot __rcu *i_dquot[MAXQUOTAS];
932
933	/* quota space reservation, managed internally by quota code */
934	qsize_t i_reserved_quota;
935	#endif
936	struct list_head dirty_list; /* dirty list for dirs and files */
937	struct list_head gdirty_list; /* linked in global dirty list */
938
939	/* linked in global inode list for cache donation */
940	struct list_head gdonate_list;
941	pgoff_t donate_start, donate_end; /* inclusive */
942	atomic_t open_count; /* # of open files */
943
944	struct task_struct *atomic_write_task; /* store atomic write task */
945	struct extent_tree *extent_tree[NR_EXTENT_CACHES];
946	/* cached extent_tree entry */
947	union {
948	struct inode *cow_inode; /* copy-on-write inode for atomic write */
949	struct inode *atomic_inode;
950	/* point to atomic_inode, available only for cow_inode */
951	};
952
953	/* avoid racing between foreground op and gc */
954	struct f2fs_rwsem i_gc_rwsem[2];
955	struct f2fs_rwsem i_xattr_sem; /* avoid racing between reading and changing EAs */
956
957	int i_extra_isize; /* size of extra space located in i_addr */
958	kprojid_t i_projid; /* id for project quota */
959	int i_inline_xattr_size; /* inline xattr size */
960	struct timespec64 i_crtime; /* inode creation time */
961	struct timespec64 i_disk_time[3];/* inode disk times */
962
963	/* for file compress */
964	atomic_t i_compr_blocks; /* # of compressed blocks */
965	unsigned char i_compress_algorithm; /* algorithm type */
966	unsigned char i_log_cluster_size; /* log of cluster size */
967	unsigned char i_compress_level; /* compress level (lz4hc,zstd) */
968	unsigned char i_compress_flag; /* compress flag */
969	unsigned int i_cluster_size; /* cluster size */
970	atomic_t writeback; /* count # of writeback thread */
971
972	unsigned int atomic_write_cnt;
973	loff_t original_i_size; /* original i_size before atomic write */
974	#ifdef CONFIG_FS_ENCRYPTION
975	struct fscrypt_inode_info *i_crypt_info; /* filesystem encryption info */
976	#endif
977	#ifdef CONFIG_FS_VERITY
978	struct fsverity_info *i_verity_info; /* filesystem verity info */
979	#endif
980	};
981
982	static inline void get_read_extent_info(struct extent_info *ext,
983	struct f2fs_extent *i_ext)
984	{
985	ext->fofs = le32_to_cpu(i_ext->fofs);
986	ext->blk = le32_to_cpu(i_ext->blk);
987	ext->len = le32_to_cpu(i_ext->len);
988	}
989
990	static inline void set_raw_read_extent(struct extent_info *ext,
991	struct f2fs_extent *i_ext)
992	{
993	i_ext->fofs = cpu_to_le32(ext->fofs);
994	i_ext->blk = cpu_to_le32(ext->blk);
995	i_ext->len = cpu_to_le32(ext->len);
996	}
997
998	static inline bool __is_discard_mergeable(struct discard_info *back,
999	struct discard_info *front, unsigned int max_len)
1000	{
1001	return (back->lstart + back->len == front->lstart) &&
1002	(back->len + front->len <= max_len);
1003	}
1004
1005	static inline bool __is_discard_back_mergeable(struct discard_info *cur,
1006	struct discard_info *back, unsigned int max_len)
1007	{
1008	return __is_discard_mergeable(back, front: cur, max_len);
1009	}
1010
1011	static inline bool __is_discard_front_mergeable(struct discard_info *cur,
1012	struct discard_info *front, unsigned int max_len)
1013	{
1014	return __is_discard_mergeable(back: cur, front, max_len);
1015	}
1016
1017	/*
1018	* For free nid management
1019	*/
1020	enum nid_state {
1021	FREE_NID, /* newly added to free nid list */
1022	PREALLOC_NID, /* it is preallocated */
1023	MAX_NID_STATE,
1024	};
1025
1026	enum nat_state {
1027	TOTAL_NAT,
1028	DIRTY_NAT,
1029	RECLAIMABLE_NAT,
1030	MAX_NAT_STATE,
1031	};
1032
1033	struct f2fs_nm_info {
1034	block_t nat_blkaddr; /* base disk address of NAT */
1035	nid_t max_nid; /* maximum possible node ids */
1036	nid_t available_nids; /* # of available node ids */
1037	nid_t next_scan_nid; /* the next nid to be scanned */
1038	nid_t max_rf_node_blocks; /* max # of nodes for recovery */
1039	unsigned int ram_thresh; /* control the memory footprint */
1040	unsigned int ra_nid_pages; /* # of nid pages to be readaheaded */
1041	unsigned int dirty_nats_ratio; /* control dirty nats ratio threshold */
1042
1043	/* NAT cache management */
1044	struct radix_tree_root nat_root;/* root of the nat entry cache */
1045	struct radix_tree_root nat_set_root;/* root of the nat set cache */
1046	struct f2fs_rwsem nat_tree_lock; /* protect nat entry tree */
1047	struct list_head nat_entries; /* cached nat entry list (clean) */
1048	spinlock_t nat_list_lock; /* protect clean nat entry list */
1049	unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */
1050	unsigned int nat_blocks; /* # of nat blocks */
1051
1052	/* free node ids management */
1053	struct radix_tree_root free_nid_root;/* root of the free_nid cache */
1054	struct list_head free_nid_list; /* list for free nids excluding preallocated nids */
1055	unsigned int nid_cnt[MAX_NID_STATE]; /* the number of free node id */
1056	spinlock_t nid_list_lock; /* protect nid lists ops */
1057	struct mutex build_lock; /* lock for build free nids */
1058	unsigned char **free_nid_bitmap;
1059	unsigned char *nat_block_bitmap;
1060	unsigned short *free_nid_count; /* free nid count of NAT block */
1061
1062	/* for checkpoint */
1063	char *nat_bitmap; /* NAT bitmap pointer */
1064
1065	unsigned int nat_bits_blocks; /* # of nat bits blocks */
1066	unsigned char *nat_bits; /* NAT bits blocks */
1067	unsigned char *full_nat_bits; /* full NAT pages */
1068	unsigned char *empty_nat_bits; /* empty NAT pages */
1069	#ifdef CONFIG_F2FS_CHECK_FS
1070	char *nat_bitmap_mir; /* NAT bitmap mirror */
1071	#endif
1072	int bitmap_size; /* bitmap size */
1073	};
1074
1075	/*
1076	* this structure is used as one of function parameters.
1077	* all the information are dedicated to a given direct node block determined
1078	* by the data offset in a file.
1079	*/
1080	struct dnode_of_data {
1081	struct inode *inode; /* vfs inode pointer */
1082	struct folio *inode_folio; /* its inode folio, NULL is possible */
1083	struct folio *node_folio; /* cached direct node folio */
1084	nid_t nid; /* node id of the direct node block */
1085	unsigned int ofs_in_node; /* data offset in the node page */
1086	bool inode_folio_locked; /* inode folio is locked or not */
1087	bool node_changed; /* is node block changed */
1088	char cur_level; /* level of hole node page */
1089	char max_level; /* level of current page located */
1090	block_t data_blkaddr; /* block address of the node block */
1091	};
1092
1093	static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode,
1094	struct folio *ifolio, struct folio *nfolio, nid_t nid)
1095	{
1096	memset(dn, 0, sizeof(*dn));
1097	dn->inode = inode;
1098	dn->inode_folio = ifolio;
1099	dn->node_folio = nfolio;
1100	dn->nid = nid;
1101	}
1102
1103	/*
1104	* For SIT manager
1105	*
1106	* By default, there are 6 active log areas across the whole main area.
1107	* When considering hot and cold data separation to reduce cleaning overhead,
1108	* we split 3 for data logs and 3 for node logs as hot, warm, and cold types,
1109	* respectively.
1110	* In the current design, you should not change the numbers intentionally.
1111	* Instead, as a mount option such as active_logs=x, you can use 2, 4, and 6
1112	* logs individually according to the underlying devices. (default: 6)
1113	* Just in case, on-disk layout covers maximum 16 logs that consist of 8 for
1114	* data and 8 for node logs.
1115	*/
1116	#define NR_CURSEG_DATA_TYPE (3)
1117	#define NR_CURSEG_NODE_TYPE (3)
1118	#define NR_CURSEG_INMEM_TYPE (2)
1119	#define NR_CURSEG_RO_TYPE (2)
1120	#define NR_CURSEG_PERSIST_TYPE (NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
1121	#define NR_CURSEG_TYPE (NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE)
1122
1123	enum log_type {
1124	CURSEG_HOT_DATA = 0, /* directory entry blocks */
1125	CURSEG_WARM_DATA, /* data blocks */
1126	CURSEG_COLD_DATA, /* multimedia or GCed data blocks */
1127	CURSEG_HOT_NODE, /* direct node blocks of directory files */
1128	CURSEG_WARM_NODE, /* direct node blocks of normal files */
1129	CURSEG_COLD_NODE, /* indirect node blocks */
1130	NR_PERSISTENT_LOG, /* number of persistent log */
1131	CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG,
1132	/* pinned file that needs consecutive block address */
1133	CURSEG_ALL_DATA_ATGC, /* SSR alloctor in hot/warm/cold data area */
1134	NO_CHECK_TYPE, /* number of persistent & inmem log */
1135	};
1136
1137	struct flush_cmd {
1138	struct completion wait;
1139	struct llist_node llnode;
1140	nid_t ino;
1141	int ret;
1142	};
1143
1144	struct flush_cmd_control {
1145	struct task_struct *f2fs_issue_flush; /* flush thread */
1146	wait_queue_head_t flush_wait_queue; /* waiting queue for wake-up */
1147	atomic_t issued_flush; /* # of issued flushes */
1148	atomic_t queued_flush; /* # of queued flushes */
1149	struct llist_head issue_list; /* list for command issue */
1150	struct llist_node *dispatch_list; /* list for command dispatch */
1151	};
1152
1153	struct f2fs_sm_info {
1154	struct sit_info *sit_info; /* whole segment information */
1155	struct free_segmap_info *free_info; /* free segment information */
1156	struct dirty_seglist_info *dirty_info; /* dirty segment information */
1157	struct curseg_info *curseg_array; /* active segment information */
1158
1159	struct f2fs_rwsem curseg_lock; /* for preventing curseg change */
1160
1161	block_t seg0_blkaddr; /* block address of 0'th segment */
1162	block_t main_blkaddr; /* start block address of main area */
1163	block_t ssa_blkaddr; /* start block address of SSA area */
1164
1165	unsigned int segment_count; /* total # of segments */
1166	unsigned int main_segments; /* # of segments in main area */
1167	unsigned int reserved_segments; /* # of reserved segments */
1168	unsigned int ovp_segments; /* # of overprovision segments */
1169
1170	/* a threshold to reclaim prefree segments */
1171	unsigned int rec_prefree_segments;
1172
1173	struct list_head sit_entry_set; /* sit entry set list */
1174
1175	unsigned int ipu_policy; /* in-place-update policy */
1176	unsigned int min_ipu_util; /* in-place-update threshold */
1177	unsigned int min_fsync_blocks; /* threshold for fsync */
1178	unsigned int min_seq_blocks; /* threshold for sequential blocks */
1179	unsigned int min_hot_blocks; /* threshold for hot block allocation */
1180	unsigned int min_ssr_sections; /* threshold to trigger SSR allocation */
1181
1182	/* for flush command control */
1183	struct flush_cmd_control *fcc_info;
1184
1185	/* for discard command control */
1186	struct discard_cmd_control *dcc_info;
1187	};
1188
1189	/*
1190	* For superblock
1191	*/
1192	/*
1193	* COUNT_TYPE for monitoring
1194	*
1195	* f2fs monitors the number of several block types such as on-writeback,
1196	* dirty dentry blocks, dirty node blocks, and dirty meta blocks.
1197	*/
1198	#define WB_DATA_TYPE(folio, f) \
1199	(f || f2fs_is_cp_guaranteed(folio) ? F2FS_WB_CP_DATA : F2FS_WB_DATA)
1200	enum count_type {
1201	F2FS_DIRTY_DENTS,
1202	F2FS_DIRTY_DATA,
1203	F2FS_DIRTY_QDATA,
1204	F2FS_DIRTY_NODES,
1205	F2FS_DIRTY_META,
1206	F2FS_DIRTY_IMETA,
1207	F2FS_WB_CP_DATA,
1208	F2FS_WB_DATA,
1209	F2FS_RD_DATA,
1210	F2FS_RD_NODE,
1211	F2FS_RD_META,
1212	F2FS_DIO_WRITE,
1213	F2FS_DIO_READ,
1214	NR_COUNT_TYPE,
1215	};
1216
1217	/*
1218	* The below are the page types of bios used in submit_bio().
1219	* The available types are:
1220	* DATA User data pages. It operates as async mode.
1221	* NODE Node pages. It operates as async mode.
1222	* META FS metadata pages such as SIT, NAT, CP.
1223	* NR_PAGE_TYPE The number of page types.
1224	* META_FLUSH Make sure the previous pages are written
1225	* with waiting the bio's completion
1226	* ... Only can be used with META.
1227	*/
1228	#define PAGE_TYPE_OF_BIO(type) ((type) > META ? META : (type))
1229	#define PAGE_TYPE_ON_MAIN(type) ((type) == DATA || (type) == NODE)
1230	enum page_type {
1231	DATA = 0,
1232	NODE = 1, /* should not change this */
1233	META,
1234	NR_PAGE_TYPE,
1235	META_FLUSH,
1236	IPU, /* the below types are used by tracepoints only. */
1237	OPU,
1238	};
1239
1240	enum temp_type {
1241	HOT = 0, /* must be zero for meta bio */
1242	WARM,
1243	COLD,
1244	NR_TEMP_TYPE,
1245	};
1246
1247	enum need_lock_type {
1248	LOCK_REQ = 0,
1249	LOCK_DONE,
1250	LOCK_RETRY,
1251	};
1252
1253	enum cp_reason_type {
1254	CP_NO_NEEDED,
1255	CP_NON_REGULAR,
1256	CP_COMPRESSED,
1257	CP_HARDLINK,
1258	CP_SB_NEED_CP,
1259	CP_WRONG_PINO,
1260	CP_NO_SPC_ROLL,
1261	CP_NODE_NEED_CP,
1262	CP_FASTBOOT_MODE,
1263	CP_SPEC_LOG_NUM,
1264	CP_RECOVER_DIR,
1265	CP_XATTR_DIR,
1266	};
1267
1268	enum iostat_type {
1269	/* WRITE IO */
1270	APP_DIRECT_IO, /* app direct write IOs */
1271	APP_BUFFERED_IO, /* app buffered write IOs */
1272	APP_WRITE_IO, /* app write IOs */
1273	APP_MAPPED_IO, /* app mapped IOs */
1274	APP_BUFFERED_CDATA_IO, /* app buffered write IOs on compressed file */
1275	APP_MAPPED_CDATA_IO, /* app mapped write IOs on compressed file */
1276	FS_DATA_IO, /* data IOs from kworker/fsync/reclaimer */
1277	FS_CDATA_IO, /* data IOs from kworker/fsync/reclaimer on compressed file */
1278	FS_NODE_IO, /* node IOs from kworker/fsync/reclaimer */
1279	FS_META_IO, /* meta IOs from kworker/reclaimer */
1280	FS_GC_DATA_IO, /* data IOs from forground gc */
1281	FS_GC_NODE_IO, /* node IOs from forground gc */
1282	FS_CP_DATA_IO, /* data IOs from checkpoint */
1283	FS_CP_NODE_IO, /* node IOs from checkpoint */
1284	FS_CP_META_IO, /* meta IOs from checkpoint */
1285
1286	/* READ IO */
1287	APP_DIRECT_READ_IO, /* app direct read IOs */
1288	APP_BUFFERED_READ_IO, /* app buffered read IOs */
1289	APP_READ_IO, /* app read IOs */
1290	APP_MAPPED_READ_IO, /* app mapped read IOs */
1291	APP_BUFFERED_CDATA_READ_IO, /* app buffered read IOs on compressed file */
1292	APP_MAPPED_CDATA_READ_IO, /* app mapped read IOs on compressed file */
1293	FS_DATA_READ_IO, /* data read IOs */
1294	FS_GDATA_READ_IO, /* data read IOs from background gc */
1295	FS_CDATA_READ_IO, /* compressed data read IOs */
1296	FS_NODE_READ_IO, /* node read IOs */
1297	FS_META_READ_IO, /* meta read IOs */
1298
1299	/* other */
1300	FS_DISCARD_IO, /* discard */
1301	FS_FLUSH_IO, /* flush */
1302	FS_ZONE_RESET_IO, /* zone reset */
1303	NR_IO_TYPE,
1304	};
1305
1306	struct f2fs_io_info {
1307	struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */
1308	nid_t ino; /* inode number */
1309	enum page_type type; /* contains DATA/NODE/META/META_FLUSH */
1310	enum temp_type temp; /* contains HOT/WARM/COLD */
1311	enum req_op op; /* contains REQ_OP_ */
1312	blk_opf_t op_flags; /* req_flag_bits */
1313	block_t new_blkaddr; /* new block address to be written */
1314	block_t old_blkaddr; /* old block address before Cow */
1315	union {
1316	struct page *page; /* page to be written */
1317	struct folio *folio;
1318	};
1319	struct page *encrypted_page; /* encrypted page */
1320	struct page *compressed_page; /* compressed page */
1321	struct list_head list; /* serialize IOs */
1322	unsigned int compr_blocks; /* # of compressed block addresses */
1323	unsigned int need_lock:8; /* indicate we need to lock cp_rwsem */
1324	unsigned int version:8; /* version of the node */
1325	unsigned int submitted:1; /* indicate IO submission */
1326	unsigned int in_list:1; /* indicate fio is in io_list */
1327	unsigned int is_por:1; /* indicate IO is from recovery or not */
1328	unsigned int encrypted:1; /* indicate file is encrypted */
1329	unsigned int meta_gc:1; /* require meta inode GC */
1330	enum iostat_type io_type; /* io type */
1331	struct writeback_control *io_wbc; /* writeback control */
1332	struct bio **bio; /* bio for ipu */
1333	sector_t *last_block; /* last block number in bio */
1334	};
1335
1336	struct bio_entry {
1337	struct bio *bio;
1338	struct list_head list;
1339	};
1340
1341	#define is_read_io(rw) ((rw) == READ)
1342	struct f2fs_bio_info {
1343	struct f2fs_sb_info *sbi; /* f2fs superblock */
1344	struct bio *bio; /* bios to merge */
1345	sector_t last_block_in_bio; /* last block number */
1346	struct f2fs_io_info fio; /* store buffered io info. */
1347	#ifdef CONFIG_BLK_DEV_ZONED
1348	struct completion zone_wait; /* condition value for the previous open zone to close */
1349	struct bio *zone_pending_bio; /* pending bio for the previous zone */
1350	void *bi_private; /* previous bi_private for pending bio */
1351	#endif
1352	struct f2fs_rwsem io_rwsem; /* blocking op for bio */
1353	spinlock_t io_lock; /* serialize DATA/NODE IOs */
1354	struct list_head io_list; /* track fios */
1355	struct list_head bio_list; /* bio entry list head */
1356	struct f2fs_rwsem bio_list_lock; /* lock to protect bio entry list */
1357	};
1358
1359	#define FDEV(i) (sbi->devs[i])
1360	#define RDEV(i) (raw_super->devs[i])
1361	struct f2fs_dev_info {
1362	struct file *bdev_file;
1363	struct block_device *bdev;
1364	char path[MAX_PATH_LEN + 1];
1365	unsigned int total_segments;
1366	block_t start_blk;
1367	block_t end_blk;
1368	#ifdef CONFIG_BLK_DEV_ZONED
1369	unsigned int nr_blkz; /* Total number of zones */
1370	unsigned long *blkz_seq; /* Bitmap indicating sequential zones */
1371	#endif
1372	};
1373
1374	enum inode_type {
1375	DIR_INODE, /* for dirty dir inode */
1376	FILE_INODE, /* for dirty regular/symlink inode */
1377	DIRTY_META, /* for all dirtied inode metadata */
1378	DONATE_INODE, /* for all inode to donate pages */
1379	NR_INODE_TYPE,
1380	};
1381
1382	/* for inner inode cache management */
1383	struct inode_management {
1384	struct radix_tree_root ino_root; /* ino entry array */
1385	spinlock_t ino_lock; /* for ino entry lock */
1386	struct list_head ino_list; /* inode list head */
1387	unsigned long ino_num; /* number of entries */
1388	};
1389
1390	/* for GC_AT */
1391	struct atgc_management {
1392	bool atgc_enabled; /* ATGC is enabled or not */
1393	struct rb_root_cached root; /* root of victim rb-tree */
1394	struct list_head victim_list; /* linked with all victim entries */
1395	unsigned int victim_count; /* victim count in rb-tree */
1396	unsigned int candidate_ratio; /* candidate ratio */
1397	unsigned int max_candidate_count; /* max candidate count */
1398	unsigned int age_weight; /* age weight, vblock_weight = 100 - age_weight */
1399	unsigned long long age_threshold; /* age threshold */
1400	};
1401
1402	struct f2fs_gc_control {
1403	unsigned int victim_segno; /* target victim segment number */
1404	int init_gc_type; /* FG_GC or BG_GC */
1405	bool no_bg_gc; /* check the space and stop bg_gc */
1406	bool should_migrate_blocks; /* should migrate blocks */
1407	bool err_gc_skipped; /* return EAGAIN if GC skipped */
1408	bool one_time; /* require one time GC in one migration unit */
1409	unsigned int nr_free_secs; /* # of free sections to do GC */
1410	};
1411
1412	/*
1413	* For s_flag in struct f2fs_sb_info
1414	* Modification on enum should be synchronized with s_flag array
1415	*/
1416	enum {
1417	SBI_IS_DIRTY, /* dirty flag for checkpoint */
1418	SBI_IS_CLOSE, /* specify unmounting */
1419	SBI_NEED_FSCK, /* need fsck.f2fs to fix */
1420	SBI_POR_DOING, /* recovery is doing or not */
1421	SBI_NEED_SB_WRITE, /* need to recover superblock */
1422	SBI_NEED_CP, /* need to checkpoint */
1423	SBI_IS_SHUTDOWN, /* shutdown by ioctl */
1424	SBI_IS_RECOVERED, /* recovered orphan/data */
1425	SBI_CP_DISABLED, /* CP was disabled last mount */
1426	SBI_CP_DISABLED_QUICK, /* CP was disabled quickly */
1427	SBI_QUOTA_NEED_FLUSH, /* need to flush quota info in CP */
1428	SBI_QUOTA_SKIP_FLUSH, /* skip flushing quota in current CP */
1429	SBI_QUOTA_NEED_REPAIR, /* quota file may be corrupted */
1430	SBI_IS_RESIZEFS, /* resizefs is in process */
1431	SBI_IS_FREEZING, /* freezefs is in process */
1432	SBI_IS_WRITABLE, /* remove ro mountoption transiently */
1433	MAX_SBI_FLAG,
1434	};
1435
1436	enum {
1437	CP_TIME,
1438	REQ_TIME,
1439	DISCARD_TIME,
1440	GC_TIME,
1441	DISABLE_TIME,
1442	ENABLE_TIME,
1443	UMOUNT_DISCARD_TIMEOUT,
1444	MAX_TIME,
1445	};
1446
1447	/* Note that you need to keep synchronization with this gc_mode_names array */
1448	enum {
1449	GC_NORMAL,
1450	GC_IDLE_CB,
1451	GC_IDLE_GREEDY,
1452	GC_IDLE_AT,
1453	GC_URGENT_HIGH,
1454	GC_URGENT_LOW,
1455	GC_URGENT_MID,
1456	MAX_GC_MODE,
1457	};
1458
1459	enum {
1460	BGGC_MODE_ON, /* background gc is on */
1461	BGGC_MODE_OFF, /* background gc is off */
1462	BGGC_MODE_SYNC, /*
1463	* background gc is on, migrating blocks
1464	* like foreground gc
1465	*/
1466	};
1467
1468	enum {
1469	FS_MODE_ADAPTIVE, /* use both lfs/ssr allocation */
1470	FS_MODE_LFS, /* use lfs allocation only */
1471	FS_MODE_FRAGMENT_SEG, /* segment fragmentation mode */
1472	FS_MODE_FRAGMENT_BLK, /* block fragmentation mode */
1473	};
1474
1475	enum {
1476	ALLOC_MODE_DEFAULT, /* stay default */
1477	ALLOC_MODE_REUSE, /* reuse segments as much as possible */
1478	};
1479
1480	enum fsync_mode {
1481	FSYNC_MODE_POSIX, /* fsync follows posix semantics */
1482	FSYNC_MODE_STRICT, /* fsync behaves in line with ext4 */
1483	FSYNC_MODE_NOBARRIER, /* fsync behaves nobarrier based on posix */
1484	};
1485
1486	enum {
1487	COMPR_MODE_FS, /*
1488	* automatically compress compression
1489	* enabled files
1490	*/
1491	COMPR_MODE_USER, /*
1492	* automatical compression is disabled.
1493	* user can control the file compression
1494	* using ioctls
1495	*/
1496	};
1497
1498	enum {
1499	DISCARD_UNIT_BLOCK, /* basic discard unit is block */
1500	DISCARD_UNIT_SEGMENT, /* basic discard unit is segment */
1501	DISCARD_UNIT_SECTION, /* basic discard unit is section */
1502	};
1503
1504	enum {
1505	MEMORY_MODE_NORMAL, /* memory mode for normal devices */
1506	MEMORY_MODE_LOW, /* memory mode for low memory devices */
1507	};
1508
1509	enum errors_option {
1510	MOUNT_ERRORS_READONLY, /* remount fs ro on errors */
1511	MOUNT_ERRORS_CONTINUE, /* continue on errors */
1512	MOUNT_ERRORS_PANIC, /* panic on errors */
1513	};
1514
1515	enum {
1516	BACKGROUND,
1517	FOREGROUND,
1518	MAX_CALL_TYPE,
1519	TOTAL_CALL = FOREGROUND,
1520	};
1521
1522	enum f2fs_lookup_mode {
1523	LOOKUP_PERF,
1524	LOOKUP_COMPAT,
1525	LOOKUP_AUTO,
1526	};
1527
1528	static inline int f2fs_test_bit(unsigned int nr, char *addr);
1529	static inline void f2fs_set_bit(unsigned int nr, char *addr);
1530	static inline void f2fs_clear_bit(unsigned int nr, char *addr);
1531
1532	/*
1533	* Layout of f2fs page.private:
1534	*
1535	* Layout A: lowest bit should be 1
1536	* | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... |
1537	* bit 0 PAGE_PRIVATE_NOT_POINTER
1538	* bit 1 PAGE_PRIVATE_ONGOING_MIGRATION
1539	* bit 2 PAGE_PRIVATE_INLINE_INODE
1540	* bit 3 PAGE_PRIVATE_REF_RESOURCE
1541	* bit 4 PAGE_PRIVATE_ATOMIC_WRITE
1542	* bit 5- f2fs private data
1543	*
1544	* Layout B: lowest bit should be 0
1545	* page.private is a wrapped pointer.
1546	*/
1547	enum {
1548	PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */
1549	PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */
1550	PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */
1551	PAGE_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */
1552	PAGE_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */
1553	PAGE_PRIVATE_MAX
1554	};
1555
1556	/* For compression */
1557	enum compress_algorithm_type {
1558	COMPRESS_LZO,
1559	COMPRESS_LZ4,
1560	COMPRESS_ZSTD,
1561	COMPRESS_LZORLE,
1562	COMPRESS_MAX,
1563	};
1564
1565	enum compress_flag {
1566	COMPRESS_CHKSUM,
1567	COMPRESS_MAX_FLAG,
1568	};
1569
1570	#define COMPRESS_WATERMARK 20
1571	#define COMPRESS_PERCENT 20
1572
1573	#define COMPRESS_DATA_RESERVED_SIZE 4
1574	struct compress_data {
1575	__le32 clen; /* compressed data size */
1576	__le32 chksum; /* compressed data checksum */
1577	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE]; /* reserved */
1578	u8 cdata[]; /* compressed data */
1579	};
1580
1581	#define COMPRESS_HEADER_SIZE (sizeof(struct compress_data))
1582
1583	#define F2FS_COMPRESSED_PAGE_MAGIC 0xF5F2C000
1584
1585	#define F2FS_ZSTD_DEFAULT_CLEVEL 1
1586
1587	#define COMPRESS_LEVEL_OFFSET 8
1588
1589	/* compress context */
1590	struct compress_ctx {
1591	struct inode *inode; /* inode the context belong to */
1592	pgoff_t cluster_idx; /* cluster index number */
1593	unsigned int cluster_size; /* page count in cluster */
1594	unsigned int log_cluster_size; /* log of cluster size */
1595	struct page **rpages; /* pages store raw data in cluster */
1596	unsigned int nr_rpages; /* total page number in rpages */
1597	struct page **cpages; /* pages store compressed data in cluster */
1598	unsigned int nr_cpages; /* total page number in cpages */
1599	unsigned int valid_nr_cpages; /* valid page number in cpages */
1600	void *rbuf; /* virtual mapped address on rpages */
1601	struct compress_data *cbuf; /* virtual mapped address on cpages */
1602	size_t rlen; /* valid data length in rbuf */
1603	size_t clen; /* valid data length in cbuf */
1604	void *private; /* payload buffer for specified compression algorithm */
1605	void *private2; /* extra payload buffer */
1606	};
1607
1608	/* compress context for write IO path */
1609	struct compress_io_ctx {
1610	u32 magic; /* magic number to indicate page is compressed */
1611	struct inode *inode; /* inode the context belong to */
1612	struct page **rpages; /* pages store raw data in cluster */
1613	unsigned int nr_rpages; /* total page number in rpages */
1614	atomic_t pending_pages; /* in-flight compressed page count */
1615	};
1616
1617	/* Context for decompressing one cluster on the read IO path */
1618	struct decompress_io_ctx {
1619	u32 magic; /* magic number to indicate page is compressed */
1620	struct inode *inode; /* inode the context belong to */
1621	struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */
1622	pgoff_t cluster_idx; /* cluster index number */
1623	unsigned int cluster_size; /* page count in cluster */
1624	unsigned int log_cluster_size; /* log of cluster size */
1625	struct page **rpages; /* pages store raw data in cluster */
1626	unsigned int nr_rpages; /* total page number in rpages */
1627	struct page **cpages; /* pages store compressed data in cluster */
1628	unsigned int nr_cpages; /* total page number in cpages */
1629	struct page **tpages; /* temp pages to pad holes in cluster */
1630	void *rbuf; /* virtual mapped address on rpages */
1631	struct compress_data *cbuf; /* virtual mapped address on cpages */
1632	size_t rlen; /* valid data length in rbuf */
1633	size_t clen; /* valid data length in cbuf */
1634
1635	/*
1636	* The number of compressed pages remaining to be read in this cluster.
1637	* This is initially nr_cpages. It is decremented by 1 each time a page
1638	* has been read (or failed to be read). When it reaches 0, the cluster
1639	* is decompressed (or an error is reported).
1640	*
1641	* If an error occurs before all the pages have been submitted for I/O,
1642	* then this will never reach 0. In this case the I/O submitter is
1643	* responsible for calling f2fs_decompress_end_io() instead.
1644	*/
1645	atomic_t remaining_pages;
1646
1647	/*
1648	* Number of references to this decompress_io_ctx.
1649	*
1650	* One reference is held for I/O completion. This reference is dropped
1651	* after the pagecache pages are updated and unlocked -- either after
1652	* decompression (and verity if enabled), or after an error.
1653	*
1654	* In addition, each compressed page holds a reference while it is in a
1655	* bio. These references are necessary prevent compressed pages from
1656	* being freed while they are still in a bio.
1657	*/
1658	refcount_t refcnt;
1659
1660	bool failed; /* IO error occurred before decompression? */
1661	bool need_verity; /* need fs-verity verification after decompression? */
1662	unsigned char compress_algorithm; /* backup algorithm type */
1663	void *private; /* payload buffer for specified decompression algorithm */
1664	void *private2; /* extra payload buffer */
1665	struct work_struct verity_work; /* work to verify the decompressed pages */
1666	struct work_struct free_work; /* work for late free this structure itself */
1667	};
1668
1669	#define NULL_CLUSTER ((unsigned int)(~0))
1670	#define MIN_COMPRESS_LOG_SIZE 2
1671	#define MAX_COMPRESS_LOG_SIZE 8
1672	#define MAX_COMPRESS_WINDOW_SIZE(log_size) ((PAGE_SIZE) << (log_size))
1673
1674	struct f2fs_sb_info {
1675	struct super_block *sb; /* pointer to VFS super block */
1676	struct proc_dir_entry *s_proc; /* proc entry */
1677	struct f2fs_super_block *raw_super; /* raw super block pointer */
1678	struct f2fs_rwsem sb_lock; /* lock for raw super block */
1679	int valid_super_block; /* valid super block no */
1680	unsigned long s_flag; /* flags for sbi */
1681	struct mutex writepages; /* mutex for writepages() */
1682
1683	#ifdef CONFIG_BLK_DEV_ZONED
1684	unsigned int blocks_per_blkz; /* F2FS blocks per zone */
1685	unsigned int unusable_blocks_per_sec; /* unusable blocks per section */
1686	unsigned int max_open_zones; /* max open zone resources of the zoned device */
1687	/* For adjust the priority writing position of data in zone UFS */
1688	unsigned int blkzone_alloc_policy;
1689	#endif
1690
1691	/* for node-related operations */
1692	struct f2fs_nm_info *nm_info; /* node manager */
1693	struct inode *node_inode; /* cache node blocks */
1694
1695	/* for segment-related operations */
1696	struct f2fs_sm_info *sm_info; /* segment manager */
1697
1698	/* for bio operations */
1699	struct f2fs_bio_info *write_io[NR_PAGE_TYPE]; /* for write bios */
1700	/* keep migration IO order for LFS mode */
1701	struct f2fs_rwsem io_order_lock;
1702	pgoff_t page_eio_ofs[NR_PAGE_TYPE]; /* EIO page offset */
1703	int page_eio_cnt[NR_PAGE_TYPE]; /* EIO count */
1704
1705	/* for checkpoint */
1706	struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */
1707	int cur_cp_pack; /* remain current cp pack */
1708	spinlock_t cp_lock; /* for flag in ckpt */
1709	struct inode *meta_inode; /* cache meta blocks */
1710	struct f2fs_rwsem cp_global_sem; /* checkpoint procedure lock */
1711	struct f2fs_rwsem cp_rwsem; /* blocking FS operations */
1712	struct f2fs_rwsem node_write; /* locking node writes */
1713	struct f2fs_rwsem node_change; /* locking node change */
1714	wait_queue_head_t cp_wait;
1715	unsigned long last_time[MAX_TIME]; /* to store time in jiffies */
1716	long interval_time[MAX_TIME]; /* to store thresholds */
1717	struct ckpt_req_control cprc_info; /* for checkpoint request control */
1718	struct cp_stats cp_stats; /* for time stat of checkpoint */
1719	struct f2fs_rwsem cp_enable_rwsem; /* block cache/dio write */
1720
1721	struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */
1722
1723	spinlock_t fsync_node_lock; /* for node entry lock */
1724	struct list_head fsync_node_list; /* node list head */
1725	unsigned int fsync_seg_id; /* sequence id */
1726	unsigned int fsync_node_num; /* number of node entries */
1727
1728	/* for orphan inode, use 0'th array */
1729	unsigned int max_orphans; /* max orphan inodes */
1730
1731	/* for inode management */
1732	struct list_head inode_list[NR_INODE_TYPE]; /* dirty inode list */
1733	spinlock_t inode_lock[NR_INODE_TYPE]; /* for dirty inode list lock */
1734	struct mutex flush_lock; /* for flush exclusion */
1735
1736	/* for extent tree cache */
1737	struct extent_tree_info extent_tree[NR_EXTENT_CACHES];
1738	atomic64_t allocated_data_blocks; /* for block age extent_cache */
1739	unsigned int max_read_extent_count; /* max read extent count per inode */
1740
1741	/* The threshold used for hot and warm data seperation*/
1742	unsigned int hot_data_age_threshold;
1743	unsigned int warm_data_age_threshold;
1744	unsigned int last_age_weight;
1745
1746	/* control donate caches */
1747	unsigned int donate_files;
1748
1749	/* basic filesystem units */
1750	unsigned int log_sectors_per_block; /* log2 sectors per block */
1751	unsigned int log_blocksize; /* log2 block size */
1752	unsigned int blocksize; /* block size */
1753	unsigned int root_ino_num; /* root inode number*/
1754	unsigned int node_ino_num; /* node inode number*/
1755	unsigned int meta_ino_num; /* meta inode number*/
1756	unsigned int log_blocks_per_seg; /* log2 blocks per segment */
1757	unsigned int blocks_per_seg; /* blocks per segment */
1758	unsigned int segs_per_sec; /* segments per section */
1759	unsigned int secs_per_zone; /* sections per zone */
1760	unsigned int total_sections; /* total section count */
1761	unsigned int total_node_count; /* total node block count */
1762	unsigned int total_valid_node_count; /* valid node block count */
1763	int dir_level; /* directory level */
1764	bool readdir_ra; /* readahead inode in readdir */
1765	u64 max_io_bytes; /* max io bytes to merge IOs */
1766
1767	block_t user_block_count; /* # of user blocks */
1768	block_t total_valid_block_count; /* # of valid blocks */
1769	block_t discard_blks; /* discard command candidats */
1770	block_t last_valid_block_count; /* for recovery */
1771	block_t reserved_blocks; /* configurable reserved blocks */
1772	block_t current_reserved_blocks; /* current reserved blocks */
1773
1774	/* Additional tracking for no checkpoint mode */
1775	block_t unusable_block_count; /* # of blocks saved by last cp */
1776
1777	unsigned int nquota_files; /* # of quota sysfile */
1778	struct f2fs_rwsem quota_sem; /* blocking cp for flags */
1779	struct task_struct *umount_lock_holder; /* s_umount lock holder */
1780
1781	/* # of pages, see count_type */
1782	atomic_t nr_pages[NR_COUNT_TYPE];
1783	/* # of allocated blocks */
1784	struct percpu_counter alloc_valid_block_count;
1785	/* # of node block writes as roll forward recovery */
1786	struct percpu_counter rf_node_block_count;
1787
1788	/* writeback control */
1789	atomic_t wb_sync_req[META]; /* count # of WB_SYNC threads */
1790
1791	/* valid inode count */
1792	struct percpu_counter total_valid_inode_count;
1793
1794	struct f2fs_mount_info mount_opt; /* mount options */
1795
1796	/* for cleaning operations */
1797	struct f2fs_rwsem gc_lock; /*
1798	* semaphore for GC, avoid
1799	* race between GC and GC or CP
1800	*/
1801	struct f2fs_gc_kthread *gc_thread; /* GC thread */
1802	struct atgc_management am; /* atgc management */
1803	unsigned int cur_victim_sec; /* current victim section num */
1804	unsigned int gc_mode; /* current GC state */
1805	unsigned int next_victim_seg[2]; /* next segment in victim section */
1806	spinlock_t gc_remaining_trials_lock;
1807	/* remaining trial count for GC_URGENT_* and GC_IDLE_* */
1808	unsigned int gc_remaining_trials;
1809
1810	/* for skip statistic */
1811	unsigned long long skipped_gc_rwsem; /* FG_GC only */
1812
1813	/* free sections reserved for pinned file */
1814	unsigned int reserved_pin_section;
1815
1816	/* threshold for gc trials on pinned files */
1817	unsigned short gc_pin_file_threshold;
1818	struct f2fs_rwsem pin_sem;
1819
1820	/* maximum # of trials to find a victim segment for SSR and GC */
1821	unsigned int max_victim_search;
1822	/* migration granularity of garbage collection, unit: segment */
1823	unsigned int migration_granularity;
1824	/* migration window granularity of garbage collection, unit: segment */
1825	unsigned int migration_window_granularity;
1826
1827	/*
1828	* for stat information.
1829	* one is for the LFS mode, and the other is for the SSR mode.
1830	*/
1831	#ifdef CONFIG_F2FS_STAT_FS
1832	struct f2fs_stat_info *stat_info; /* FS status information */
1833	atomic_t meta_count[META_MAX]; /* # of meta blocks */
1834	unsigned int segment_count[2]; /* # of allocated segments */
1835	unsigned int block_count[2]; /* # of allocated blocks */
1836	atomic_t inplace_count; /* # of inplace update */
1837	/* # of lookup extent cache */
1838	atomic64_t total_hit_ext[NR_EXTENT_CACHES];
1839	/* # of hit rbtree extent node */
1840	atomic64_t read_hit_rbtree[NR_EXTENT_CACHES];
1841	/* # of hit cached extent node */
1842	atomic64_t read_hit_cached[NR_EXTENT_CACHES];
1843	/* # of hit largest extent node in read extent cache */
1844	atomic64_t read_hit_largest;
1845	atomic_t inline_xattr; /* # of inline_xattr inodes */
1846	atomic_t inline_inode; /* # of inline_data inodes */
1847	atomic_t inline_dir; /* # of inline_dentry inodes */
1848	atomic_t compr_inode; /* # of compressed inodes */
1849	atomic64_t compr_blocks; /* # of compressed blocks */
1850	atomic_t swapfile_inode; /* # of swapfile inodes */
1851	atomic_t atomic_files; /* # of opened atomic file */
1852	atomic_t max_aw_cnt; /* max # of atomic writes */
1853	unsigned int io_skip_bggc; /* skip background gc for in-flight IO */
1854	unsigned int other_skip_bggc; /* skip background gc for other reasons */
1855	unsigned int ndirty_inode[NR_INODE_TYPE]; /* # of dirty inodes */
1856	atomic_t cp_call_count[MAX_CALL_TYPE]; /* # of cp call */
1857	#endif
1858	spinlock_t stat_lock; /* lock for stat operations */
1859
1860	/* to attach REQ_META|REQ_FUA flags */
1861	unsigned int data_io_flag;
1862	unsigned int node_io_flag;
1863
1864	/* For sysfs support */
1865	struct kobject s_kobj; /* /sys/fs/f2fs/<devname> */
1866	struct completion s_kobj_unregister;
1867
1868	struct kobject s_stat_kobj; /* /sys/fs/f2fs/<devname>/stat */
1869	struct completion s_stat_kobj_unregister;
1870
1871	struct kobject s_feature_list_kobj; /* /sys/fs/f2fs/<devname>/feature_list */
1872	struct completion s_feature_list_kobj_unregister;
1873
1874	/* For shrinker support */
1875	struct list_head s_list;
1876	struct mutex umount_mutex;
1877	unsigned int shrinker_run_no;
1878
1879	/* For multi devices */
1880	int s_ndevs; /* number of devices */
1881	struct f2fs_dev_info *devs; /* for device list */
1882	unsigned int dirty_device; /* for checkpoint data flush */
1883	spinlock_t dev_lock; /* protect dirty_device */
1884	bool aligned_blksize; /* all devices has the same logical blksize */
1885	unsigned int first_seq_zone_segno; /* first segno in sequential zone */
1886	unsigned int bggc_io_aware; /* For adjust the BG_GC priority when pending IO */
1887	unsigned int allocate_section_hint; /* the boundary position between devices */
1888	unsigned int allocate_section_policy; /* determine the section writing priority */
1889
1890	/* For write statistics */
1891	u64 sectors_written_start;
1892	u64 kbytes_written;
1893
1894	/* Precomputed FS UUID checksum for seeding other checksums */
1895	__u32 s_chksum_seed;
1896
1897	struct workqueue_struct *post_read_wq; /* post read workqueue */
1898
1899	/*
1900	* If we are in irq context, let's update error information into
1901	* on-disk superblock in the work.
1902	*/
1903	struct work_struct s_error_work;
1904	unsigned char errors[MAX_F2FS_ERRORS]; /* error flags */
1905	unsigned char stop_reason[MAX_STOP_REASON]; /* stop reason */
1906	spinlock_t error_lock; /* protect errors/stop_reason array */
1907	bool error_dirty; /* errors of sb is dirty */
1908
1909	/* For reclaimed segs statistics per each GC mode */
1910	unsigned int gc_segment_mode; /* GC state for reclaimed segments */
1911	unsigned int gc_reclaimed_segs[MAX_GC_MODE]; /* Reclaimed segs for each mode */
1912
1913	unsigned long seq_file_ra_mul; /* multiplier for ra_pages of seq. files in fadvise */
1914
1915	int max_fragment_chunk; /* max chunk size for block fragmentation mode */
1916	int max_fragment_hole; /* max hole size for block fragmentation mode */
1917
1918	/* For atomic write statistics */
1919	atomic64_t current_atomic_write;
1920	s64 peak_atomic_write;
1921	u64 committed_atomic_block;
1922	u64 revoked_atomic_block;
1923
1924	/* carve out reserved_blocks from total blocks */
1925	bool carve_out;
1926
1927	#ifdef CONFIG_F2FS_FS_COMPRESSION
1928	struct kmem_cache *page_array_slab; /* page array entry */
1929	unsigned int page_array_slab_size; /* default page array slab size */
1930
1931	/* For runtime compression statistics */
1932	u64 compr_written_block;
1933	u64 compr_saved_block;
1934	u32 compr_new_inode;
1935
1936	/* For compressed block cache */
1937	struct inode *compress_inode; /* cache compressed blocks */
1938	unsigned int compress_percent; /* cache page percentage */
1939	unsigned int compress_watermark; /* cache page watermark */
1940	atomic_t compress_page_hit; /* cache hit count */
1941	#endif
1942
1943	#ifdef CONFIG_F2FS_IOSTAT
1944	/* For app/fs IO statistics */
1945	spinlock_t iostat_lock;
1946	unsigned long long iostat_count[NR_IO_TYPE];
1947	unsigned long long iostat_bytes[NR_IO_TYPE];
1948	unsigned long long prev_iostat_bytes[NR_IO_TYPE];
1949	bool iostat_enable;
1950	unsigned long iostat_next_period;
1951	unsigned int iostat_period_ms;
1952
1953	/* For io latency related statistics info in one iostat period */
1954	spinlock_t iostat_lat_lock;
1955	struct iostat_lat_info *iostat_io_lat;
1956	#endif
1957	};
1958
1959	/* Definitions to access f2fs_sb_info */
1960	#define SEGS_TO_BLKS(sbi, segs) \
1961	((segs) << (sbi)->log_blocks_per_seg)
1962	#define BLKS_TO_SEGS(sbi, blks) \
1963	((blks) >> (sbi)->log_blocks_per_seg)
1964
1965	#define BLKS_PER_SEG(sbi) ((sbi)->blocks_per_seg)
1966	#define BLKS_PER_SEC(sbi) (SEGS_TO_BLKS(sbi, (sbi)->segs_per_sec))
1967	#define SEGS_PER_SEC(sbi) ((sbi)->segs_per_sec)
1968
1969	__printf(3, 4)
1970	void f2fs_printk(struct f2fs_sb_info *sbi, bool limit_rate, const char *fmt, ...);
1971
1972	#define f2fs_err(sbi, fmt, ...) \
1973	f2fs_printk(sbi, false, KERN_ERR fmt, ##__VA_ARGS__)
1974	#define f2fs_warn(sbi, fmt, ...) \
1975	f2fs_printk(sbi, false, KERN_WARNING fmt, ##__VA_ARGS__)
1976	#define f2fs_notice(sbi, fmt, ...) \
1977	f2fs_printk(sbi, false, KERN_NOTICE fmt, ##__VA_ARGS__)
1978	#define f2fs_info(sbi, fmt, ...) \
1979	f2fs_printk(sbi, false, KERN_INFO fmt, ##__VA_ARGS__)
1980	#define f2fs_debug(sbi, fmt, ...) \
1981	f2fs_printk(sbi, false, KERN_DEBUG fmt, ##__VA_ARGS__)
1982
1983	#define f2fs_err_ratelimited(sbi, fmt, ...) \
1984	f2fs_printk(sbi, true, KERN_ERR fmt, ##__VA_ARGS__)
1985	#define f2fs_warn_ratelimited(sbi, fmt, ...) \
1986	f2fs_printk(sbi, true, KERN_WARNING fmt, ##__VA_ARGS__)
1987	#define f2fs_info_ratelimited(sbi, fmt, ...) \
1988	f2fs_printk(sbi, true, KERN_INFO fmt, ##__VA_ARGS__)
1989
1990	#ifdef CONFIG_F2FS_FAULT_INJECTION
1991	#define time_to_inject(sbi, type) __time_to_inject(sbi, type, __func__, \
1992	__builtin_return_address(0))
1993	static inline bool __time_to_inject(struct f2fs_sb_info *sbi, int type,
1994	const char *func, const char *parent_func)
1995	{
1996	struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info;
1997
1998	if (!ffi->inject_rate)
1999	return false;
2000
2001	if (!IS_FAULT_SET(ffi, type))
2002	return false;
2003
2004	atomic_inc(v: &ffi->inject_ops);
2005	if (atomic_read(v: &ffi->inject_ops) >= ffi->inject_rate) {
2006	atomic_set(v: &ffi->inject_ops, i: 0);
2007	ffi->inject_count[type]++;
2008	f2fs_info_ratelimited(sbi, "inject %s in %s of %pS",
2009	f2fs_fault_name[type], func, parent_func);
2010	return true;
2011	}
2012	return false;
2013	}
2014	#else
2015	static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
2016	{
2017	return false;
2018	}
2019	#endif
2020
2021	/*
2022	* Test if the mounted volume is a multi-device volume.
2023	* - For a single regular disk volume, sbi->s_ndevs is 0.
2024	* - For a single zoned disk volume, sbi->s_ndevs is 1.
2025	* - For a multi-device volume, sbi->s_ndevs is always 2 or more.
2026	*/
2027	static inline bool f2fs_is_multi_device(struct f2fs_sb_info *sbi)
2028	{
2029	return sbi->s_ndevs > 1;
2030	}
2031
2032	static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
2033	{
2034	unsigned long now = jiffies;
2035
2036	sbi->last_time[type] = now;
2037
2038	/* DISCARD_TIME and GC_TIME are based on REQ_TIME */
2039	if (type == REQ_TIME) {
2040	sbi->last_time[DISCARD_TIME] = now;
2041	sbi->last_time[GC_TIME] = now;
2042	}
2043	}
2044
2045	static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type)
2046	{
2047	unsigned long interval = sbi->interval_time[type] * HZ;
2048
2049	return time_after(jiffies, sbi->last_time[type] + interval);
2050	}
2051
2052	static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi,
2053	int type)
2054	{
2055	unsigned long interval = sbi->interval_time[type] * HZ;
2056	unsigned int wait_ms = 0;
2057	long delta;
2058
2059	delta = (sbi->last_time[type] + interval) - jiffies;
2060	if (delta > 0)
2061	wait_ms = jiffies_to_msecs(j: delta);
2062
2063	return wait_ms;
2064	}
2065
2066	/*
2067	* Inline functions
2068	*/
2069	static inline u32 __f2fs_crc32(u32 crc, const void *address,
2070	unsigned int length)
2071	{
2072	return crc32(crc, p: address, len: length);
2073	}
2074
2075	static inline u32 f2fs_crc32(const void *address, unsigned int length)
2076	{
2077	return __f2fs_crc32(F2FS_SUPER_MAGIC, address, length);
2078	}
2079
2080	static inline u32 f2fs_chksum(u32 crc, const void *address, unsigned int length)
2081	{
2082	return __f2fs_crc32(crc, address, length);
2083	}
2084
2085	static inline struct f2fs_inode_info *F2FS_I(struct inode *inode)
2086	{
2087	return container_of(inode, struct f2fs_inode_info, vfs_inode);
2088	}
2089
2090	static inline struct f2fs_sb_info *F2FS_SB(struct super_block *sb)
2091	{
2092	return sb->s_fs_info;
2093	}
2094
2095	static inline struct f2fs_sb_info *F2FS_I_SB(struct inode *inode)
2096	{
2097	return F2FS_SB(sb: inode->i_sb);
2098	}
2099
2100	static inline struct f2fs_sb_info *F2FS_M_SB(struct address_space *mapping)
2101	{
2102	return F2FS_I_SB(inode: mapping->host);
2103	}
2104
2105	static inline struct f2fs_sb_info *F2FS_F_SB(const struct folio *folio)
2106	{
2107	return F2FS_M_SB(mapping: folio->mapping);
2108	}
2109
2110	static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi)
2111	{
2112	return (struct f2fs_super_block *)(sbi->raw_super);
2113	}
2114
2115	static inline struct f2fs_super_block *F2FS_SUPER_BLOCK(struct folio *folio,
2116	pgoff_t index)
2117	{
2118	pgoff_t idx_in_folio = index % folio_nr_pages(folio);
2119
2120	return (struct f2fs_super_block *)
2121	(page_address(folio_page(folio, idx_in_folio)) +
2122	F2FS_SUPER_OFFSET);
2123	}
2124
2125	static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi)
2126	{
2127	return (struct f2fs_checkpoint *)(sbi->ckpt);
2128	}
2129
2130	static inline struct f2fs_node *F2FS_NODE(const struct folio *folio)
2131	{
2132	return (struct f2fs_node *)folio_address(folio);
2133	}
2134
2135	static inline struct f2fs_inode *F2FS_INODE(const struct folio *folio)
2136	{
2137	return &((struct f2fs_node *)folio_address(folio))->i;
2138	}
2139
2140	static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi)
2141	{
2142	return (struct f2fs_nm_info *)(sbi->nm_info);
2143	}
2144
2145	static inline struct f2fs_sm_info *SM_I(struct f2fs_sb_info *sbi)
2146	{
2147	return (struct f2fs_sm_info *)(sbi->sm_info);
2148	}
2149
2150	static inline struct sit_info *SIT_I(struct f2fs_sb_info *sbi)
2151	{
2152	return (struct sit_info *)(SM_I(sbi)->sit_info);
2153	}
2154
2155	static inline struct free_segmap_info *FREE_I(struct f2fs_sb_info *sbi)
2156	{
2157	return (struct free_segmap_info *)(SM_I(sbi)->free_info);
2158	}
2159
2160	static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi)
2161	{
2162	return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info);
2163	}
2164
2165	static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi)
2166	{
2167	return sbi->meta_inode->i_mapping;
2168	}
2169
2170	static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi)
2171	{
2172	return sbi->node_inode->i_mapping;
2173	}
2174
2175	static inline bool is_meta_folio(struct folio *folio)
2176	{
2177	return folio->mapping == META_MAPPING(sbi: F2FS_F_SB(folio));
2178	}
2179
2180	static inline bool is_node_folio(struct folio *folio)
2181	{
2182	return folio->mapping == NODE_MAPPING(sbi: F2FS_F_SB(folio));
2183	}
2184
2185	static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type)
2186	{
2187	return test_bit(type, &sbi->s_flag);
2188	}
2189
2190	static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
2191	{
2192	set_bit(nr: type, addr: &sbi->s_flag);
2193	}
2194
2195	static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
2196	{
2197	clear_bit(nr: type, addr: &sbi->s_flag);
2198	}
2199
2200	static inline unsigned long long cur_cp_version(struct f2fs_checkpoint *cp)
2201	{
2202	return le64_to_cpu(cp->checkpoint_ver);
2203	}
2204
2205	static inline unsigned long f2fs_qf_ino(struct super_block *sb, int type)
2206	{
2207	if (type < F2FS_MAX_QUOTAS)
2208	return le32_to_cpu(F2FS_SB(sb)->raw_super->qf_ino[type]);
2209	return 0;
2210	}
2211
2212	static inline __u64 cur_cp_crc(struct f2fs_checkpoint *cp)
2213	{
2214	size_t crc_offset = le32_to_cpu(cp->checksum_offset);
2215	return le32_to_cpu(*((__le32 *)((unsigned char *)cp + crc_offset)));
2216	}
2217
2218	static inline bool __is_set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2219	{
2220	unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2221
2222	return ckpt_flags & f;
2223	}
2224
2225	static inline bool is_set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2226	{
2227	return __is_set_ckpt_flags(cp: F2FS_CKPT(sbi), f);
2228	}
2229
2230	static inline void __set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2231	{
2232	unsigned int ckpt_flags;
2233
2234	ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2235	ckpt_flags |= f;
2236	cp->ckpt_flags = cpu_to_le32(ckpt_flags);
2237	}
2238
2239	static inline void set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2240	{
2241	unsigned long flags;
2242
2243	spin_lock_irqsave(&sbi->cp_lock, flags);
2244	__set_ckpt_flags(cp: F2FS_CKPT(sbi), f);
2245	spin_unlock_irqrestore(lock: &sbi->cp_lock, flags);
2246	}
2247
2248	static inline void __clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2249	{
2250	unsigned int ckpt_flags;
2251
2252	ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2253	ckpt_flags &= (~f);
2254	cp->ckpt_flags = cpu_to_le32(ckpt_flags);
2255	}
2256
2257	static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2258	{
2259	unsigned long flags;
2260
2261	spin_lock_irqsave(&sbi->cp_lock, flags);
2262	__clear_ckpt_flags(cp: F2FS_CKPT(sbi), f);
2263	spin_unlock_irqrestore(lock: &sbi->cp_lock, flags);
2264	}
2265
2266	#define init_f2fs_rwsem(sem) \
2267	do { \
2268	static struct lock_class_key __key; \
2269	\
2270	__init_f2fs_rwsem((sem), #sem, &__key); \
2271	} while (0)
2272
2273	static inline void __init_f2fs_rwsem(struct f2fs_rwsem *sem,
2274	const char *sem_name, struct lock_class_key *key)
2275	{
2276	__init_rwsem(sem: &sem->internal_rwsem, name: sem_name, key);
2277	#ifdef CONFIG_F2FS_UNFAIR_RWSEM
2278	init_waitqueue_head(&sem->read_waiters);
2279	#endif
2280	}
2281
2282	static inline int f2fs_rwsem_is_locked(struct f2fs_rwsem *sem)
2283	{
2284	return rwsem_is_locked(sem: &sem->internal_rwsem);
2285	}
2286
2287	static inline int f2fs_rwsem_is_contended(struct f2fs_rwsem *sem)
2288	{
2289	return rwsem_is_contended(sem: &sem->internal_rwsem);
2290	}
2291
2292	static inline void f2fs_down_read(struct f2fs_rwsem *sem)
2293	{
2294	#ifdef CONFIG_F2FS_UNFAIR_RWSEM
2295	wait_event(sem->read_waiters, down_read_trylock(&sem->internal_rwsem));
2296	#else
2297	down_read(&sem->internal_rwsem);
2298	#endif
2299	}
2300
2301	static inline int f2fs_down_read_trylock(struct f2fs_rwsem *sem)
2302	{
2303	return down_read_trylock(sem: &sem->internal_rwsem);
2304	}
2305
2306	static inline void f2fs_up_read(struct f2fs_rwsem *sem)
2307	{
2308	up_read(sem: &sem->internal_rwsem);
2309	}
2310
2311	static inline void f2fs_down_write(struct f2fs_rwsem *sem)
2312	{
2313	down_write(sem: &sem->internal_rwsem);
2314	}
2315
2316	#ifdef CONFIG_DEBUG_LOCK_ALLOC
2317	static inline void f2fs_down_read_nested(struct f2fs_rwsem *sem, int subclass)
2318	{
2319	down_read_nested(sem: &sem->internal_rwsem, subclass);
2320	}
2321
2322	static inline void f2fs_down_write_nested(struct f2fs_rwsem *sem, int subclass)
2323	{
2324	down_write_nested(sem: &sem->internal_rwsem, subclass);
2325	}
2326	#else
2327	#define f2fs_down_read_nested(sem, subclass) f2fs_down_read(sem)
2328	#define f2fs_down_write_nested(sem, subclass) f2fs_down_write(sem)
2329	#endif
2330
2331	static inline int f2fs_down_write_trylock(struct f2fs_rwsem *sem)
2332	{
2333	return down_write_trylock(sem: &sem->internal_rwsem);
2334	}
2335
2336	static inline void f2fs_up_write(struct f2fs_rwsem *sem)
2337	{
2338	up_write(sem: &sem->internal_rwsem);
2339	#ifdef CONFIG_F2FS_UNFAIR_RWSEM
2340	wake_up_all(&sem->read_waiters);
2341	#endif
2342	}
2343
2344	static inline void disable_nat_bits(struct f2fs_sb_info *sbi, bool lock)
2345	{
2346	unsigned long flags;
2347	unsigned char *nat_bits;
2348
2349	/*
2350	* In order to re-enable nat_bits we need to call fsck.f2fs by
2351	* set_sbi_flag(sbi, SBI_NEED_FSCK). But it may give huge cost,
2352	* so let's rely on regular fsck or unclean shutdown.
2353	*/
2354
2355	if (lock)
2356	spin_lock_irqsave(&sbi->cp_lock, flags);
2357	__clear_ckpt_flags(cp: F2FS_CKPT(sbi), CP_NAT_BITS_FLAG);
2358	nat_bits = NM_I(sbi)->nat_bits;
2359	NM_I(sbi)->nat_bits = NULL;
2360	if (lock)
2361	spin_unlock_irqrestore(lock: &sbi->cp_lock, flags);
2362
2363	kvfree(addr: nat_bits);
2364	}
2365
2366	static inline bool enabled_nat_bits(struct f2fs_sb_info *sbi,
2367	struct cp_control *cpc)
2368	{
2369	bool set = is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
2370
2371	return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set;
2372	}
2373
2374	static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
2375	{
2376	f2fs_down_read(sem: &sbi->cp_rwsem);
2377	}
2378
2379	static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi)
2380	{
2381	if (time_to_inject(sbi, FAULT_LOCK_OP))
2382	return 0;
2383	return f2fs_down_read_trylock(sem: &sbi->cp_rwsem);
2384	}
2385
2386	static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi)
2387	{
2388	f2fs_up_read(sem: &sbi->cp_rwsem);
2389	}
2390
2391	static inline void f2fs_lock_all(struct f2fs_sb_info *sbi)
2392	{
2393	f2fs_down_write(sem: &sbi->cp_rwsem);
2394	}
2395
2396	static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi)
2397	{
2398	f2fs_up_write(sem: &sbi->cp_rwsem);
2399	}
2400
2401	static inline int __get_cp_reason(struct f2fs_sb_info *sbi)
2402	{
2403	int reason = CP_SYNC;
2404
2405	if (test_opt(sbi, FASTBOOT))
2406	reason = CP_FASTBOOT;
2407	if (is_sbi_flag_set(sbi, type: SBI_IS_CLOSE))
2408	reason = CP_UMOUNT;
2409	return reason;
2410	}
2411
2412	static inline bool __remain_node_summaries(int reason)
2413	{
2414	return (reason & (CP_UMOUNT | CP_FASTBOOT));
2415	}
2416
2417	static inline bool __exist_node_summaries(struct f2fs_sb_info *sbi)
2418	{
2419	return (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG) ||
2420	is_set_ckpt_flags(sbi, CP_FASTBOOT_FLAG));
2421	}
2422
2423	/*
2424	* Check whether the inode has blocks or not
2425	*/
2426	static inline int F2FS_HAS_BLOCKS(struct inode *inode)
2427	{
2428	block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0;
2429
2430	return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block;
2431	}
2432
2433	static inline bool f2fs_has_xattr_block(unsigned int ofs)
2434	{
2435	return ofs == XATTR_NODE_OFFSET;
2436	}
2437
2438	static inline bool __allow_reserved_root(struct f2fs_sb_info *sbi,
2439	struct inode *inode, bool cap)
2440	{
2441	if (!inode)
2442	return true;
2443	if (IS_NOQUOTA(inode))
2444	return true;
2445	if (uid_eq(F2FS_OPTION(sbi).s_resuid, current_fsuid()))
2446	return true;
2447	if (!gid_eq(F2FS_OPTION(sbi).s_resgid, GLOBAL_ROOT_GID) &&
2448	in_group_p(F2FS_OPTION(sbi).s_resgid))
2449	return true;
2450	if (cap && capable(CAP_SYS_RESOURCE))
2451	return true;
2452	return false;
2453	}
2454
2455	static inline unsigned int get_available_block_count(struct f2fs_sb_info *sbi,
2456	struct inode *inode, bool cap)
2457	{
2458	block_t avail_user_block_count;
2459
2460	avail_user_block_count = sbi->user_block_count -
2461	sbi->current_reserved_blocks;
2462
2463	if (test_opt(sbi, RESERVE_ROOT) && !__allow_reserved_root(sbi, inode, cap))
2464	avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
2465
2466	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2467	if (avail_user_block_count > sbi->unusable_block_count)
2468	avail_user_block_count -= sbi->unusable_block_count;
2469	else
2470	avail_user_block_count = 0;
2471	}
2472
2473	return avail_user_block_count;
2474	}
2475
2476	static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool);
2477	static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
2478	struct inode *inode, blkcnt_t *count, bool partial)
2479	{
2480	long long diff = 0, release = 0;
2481	block_t avail_user_block_count;
2482	int ret;
2483
2484	ret = dquot_reserve_block(inode, nr: *count);
2485	if (ret)
2486	return ret;
2487
2488	if (time_to_inject(sbi, FAULT_BLOCK)) {
2489	release = *count;
2490	goto release_quota;
2491	}
2492
2493	/*
2494	* let's increase this in prior to actual block count change in order
2495	* for f2fs_sync_file to avoid data races when deciding checkpoint.
2496	*/
2497	percpu_counter_add(fbc: &sbi->alloc_valid_block_count, amount: (*count));
2498
2499	spin_lock(lock: &sbi->stat_lock);
2500
2501	avail_user_block_count = get_available_block_count(sbi, inode, cap: true);
2502	diff = (long long)sbi->total_valid_block_count + *count -
2503	avail_user_block_count;
2504	if (unlikely(diff > 0)) {
2505	if (!partial) {
2506	spin_unlock(lock: &sbi->stat_lock);
2507	release = *count;
2508	goto enospc;
2509	}
2510	if (diff > *count)
2511	diff = *count;
2512	*count -= diff;
2513	release = diff;
2514	if (!*count) {
2515	spin_unlock(lock: &sbi->stat_lock);
2516	goto enospc;
2517	}
2518	}
2519	sbi->total_valid_block_count += (block_t)(*count);
2520
2521	spin_unlock(lock: &sbi->stat_lock);
2522
2523	if (unlikely(release)) {
2524	percpu_counter_sub(fbc: &sbi->alloc_valid_block_count, amount: release);
2525	dquot_release_reservation_block(inode, nr: release);
2526	}
2527	f2fs_i_blocks_write(inode, *count, true, true);
2528	return 0;
2529
2530	enospc:
2531	percpu_counter_sub(fbc: &sbi->alloc_valid_block_count, amount: release);
2532	release_quota:
2533	dquot_release_reservation_block(inode, nr: release);
2534	return -ENOSPC;
2535	}
2536
2537	#define PAGE_PRIVATE_GET_FUNC(name, flagname) \
2538	static inline bool folio_test_f2fs_##name(const struct folio *folio) \
2539	{ \
2540	unsigned long priv = (unsigned long)folio->private; \
2541	unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \
2542	(1UL << PAGE_PRIVATE_##flagname); \
2543	return (priv & v) == v; \
2544	} \
2545	static inline bool page_private_##name(struct page *page) \
2546	{ \
2547	return PagePrivate(page) && \
2548	test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \
2549	test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
2550	}
2551
2552	#define PAGE_PRIVATE_SET_FUNC(name, flagname) \
2553	static inline void folio_set_f2fs_##name(struct folio *folio) \
2554	{ \
2555	unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \
2556	(1UL << PAGE_PRIVATE_##flagname); \
2557	if (!folio->private) \
2558	folio_attach_private(folio, (void *)v); \
2559	else { \
2560	v |= (unsigned long)folio->private; \
2561	folio->private = (void *)v; \
2562	} \
2563	} \
2564	static inline void set_page_private_##name(struct page *page) \
2565	{ \
2566	if (!PagePrivate(page)) \
2567	attach_page_private(page, (void *)0); \
2568	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \
2569	set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
2570	}
2571
2572	#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \
2573	static inline void folio_clear_f2fs_##name(struct folio *folio) \
2574	{ \
2575	unsigned long v = (unsigned long)folio->private; \
2576	\
2577	v &= ~(1UL << PAGE_PRIVATE_##flagname); \
2578	if (v == (1UL << PAGE_PRIVATE_NOT_POINTER)) \
2579	folio_detach_private(folio); \
2580	else \
2581	folio->private = (void *)v; \
2582	} \
2583	static inline void clear_page_private_##name(struct page *page) \
2584	{ \
2585	clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
2586	if (page_private(page) == BIT(PAGE_PRIVATE_NOT_POINTER)) \
2587	detach_page_private(page); \
2588	}
2589
2590	PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
2591	PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE);
2592	PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
2593	PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE);
2594
2595	PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
2596	PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE);
2597	PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
2598	PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE);
2599
2600	PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
2601	PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE);
2602	PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
2603	PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
2604
2605	static inline unsigned long folio_get_f2fs_data(struct folio *folio)
2606	{
2607	unsigned long data = (unsigned long)folio->private;
2608
2609	if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
2610	return 0;
2611	return data >> PAGE_PRIVATE_MAX;
2612	}
2613
2614	static inline void folio_set_f2fs_data(struct folio *folio, unsigned long data)
2615	{
2616	data = (1UL << PAGE_PRIVATE_NOT_POINTER) | (data << PAGE_PRIVATE_MAX);
2617
2618	if (!folio_test_private(folio))
2619	folio_attach_private(folio, data: (void *)data);
2620	else
2621	folio->private = (void *)((unsigned long)folio->private | data);
2622	}
2623
2624	static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
2625	struct inode *inode,
2626	block_t count)
2627	{
2628	blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK;
2629
2630	spin_lock(lock: &sbi->stat_lock);
2631	if (unlikely(sbi->total_valid_block_count < count)) {
2632	f2fs_warn(sbi, "Inconsistent total_valid_block_count:%u, ino:%lu, count:%u",
2633	sbi->total_valid_block_count, inode->i_ino, count);
2634	sbi->total_valid_block_count = 0;
2635	set_sbi_flag(sbi, type: SBI_NEED_FSCK);
2636	} else {
2637	sbi->total_valid_block_count -= count;
2638	}
2639	if (sbi->reserved_blocks &&
2640	sbi->current_reserved_blocks < sbi->reserved_blocks)
2641	sbi->current_reserved_blocks = min(sbi->reserved_blocks,
2642	sbi->current_reserved_blocks + count);
2643	spin_unlock(lock: &sbi->stat_lock);
2644	if (unlikely(inode->i_blocks < sectors)) {
2645	f2fs_warn(sbi, "Inconsistent i_blocks, ino:%lu, iblocks:%llu, sectors:%llu",
2646	inode->i_ino,
2647	(unsigned long long)inode->i_blocks,
2648	(unsigned long long)sectors);
2649	set_sbi_flag(sbi, type: SBI_NEED_FSCK);
2650	return;
2651	}
2652	f2fs_i_blocks_write(inode, count, false, true);
2653	}
2654
2655	static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
2656	{
2657	atomic_inc(v: &sbi->nr_pages[count_type]);
2658
2659	if (count_type == F2FS_DIRTY_DENTS ||
2660	count_type == F2FS_DIRTY_NODES ||
2661	count_type == F2FS_DIRTY_META ||
2662	count_type == F2FS_DIRTY_QDATA ||
2663	count_type == F2FS_DIRTY_IMETA)
2664	set_sbi_flag(sbi, type: SBI_IS_DIRTY);
2665	}
2666
2667	static inline void inode_inc_dirty_pages(struct inode *inode)
2668	{
2669	atomic_inc(v: &F2FS_I(inode)->dirty_pages);
2670	inc_page_count(sbi: F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2671	F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2672	if (IS_NOQUOTA(inode))
2673	inc_page_count(sbi: F2FS_I_SB(inode), count_type: F2FS_DIRTY_QDATA);
2674	}
2675
2676	static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
2677	{
2678	atomic_dec(v: &sbi->nr_pages[count_type]);
2679	}
2680
2681	static inline void inode_dec_dirty_pages(struct inode *inode)
2682	{
2683	if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
2684	!S_ISLNK(inode->i_mode))
2685	return;
2686
2687	atomic_dec(v: &F2FS_I(inode)->dirty_pages);
2688	dec_page_count(sbi: F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2689	F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2690	if (IS_NOQUOTA(inode))
2691	dec_page_count(sbi: F2FS_I_SB(inode), count_type: F2FS_DIRTY_QDATA);
2692	}
2693
2694	static inline void inc_atomic_write_cnt(struct inode *inode)
2695	{
2696	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2697	struct f2fs_inode_info *fi = F2FS_I(inode);
2698	u64 current_write;
2699
2700	fi->atomic_write_cnt++;
2701	atomic64_inc(v: &sbi->current_atomic_write);
2702	current_write = atomic64_read(v: &sbi->current_atomic_write);
2703	if (current_write > sbi->peak_atomic_write)
2704	sbi->peak_atomic_write = current_write;
2705	}
2706
2707	static inline void release_atomic_write_cnt(struct inode *inode)
2708	{
2709	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2710	struct f2fs_inode_info *fi = F2FS_I(inode);
2711
2712	atomic64_sub(i: fi->atomic_write_cnt, v: &sbi->current_atomic_write);
2713	fi->atomic_write_cnt = 0;
2714	}
2715
2716	static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
2717	{
2718	return atomic_read(v: &sbi->nr_pages[count_type]);
2719	}
2720
2721	static inline int get_dirty_pages(struct inode *inode)
2722	{
2723	return atomic_read(v: &F2FS_I(inode)->dirty_pages);
2724	}
2725
2726	static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type)
2727	{
2728	return div_u64(dividend: get_pages(sbi, count_type: block_type) + BLKS_PER_SEC(sbi) - 1,
2729	BLKS_PER_SEC(sbi));
2730	}
2731
2732	static inline block_t valid_user_blocks(struct f2fs_sb_info *sbi)
2733	{
2734	return sbi->total_valid_block_count;
2735	}
2736
2737	static inline block_t discard_blocks(struct f2fs_sb_info *sbi)
2738	{
2739	return sbi->discard_blks;
2740	}
2741
2742	static inline unsigned long __bitmap_size(struct f2fs_sb_info *sbi, int flag)
2743	{
2744	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2745
2746	/* return NAT or SIT bitmap */
2747	if (flag == NAT_BITMAP)
2748	return le32_to_cpu(ckpt->nat_ver_bitmap_bytesize);
2749	else if (flag == SIT_BITMAP)
2750	return le32_to_cpu(ckpt->sit_ver_bitmap_bytesize);
2751
2752	return 0;
2753	}
2754
2755	static inline block_t __cp_payload(struct f2fs_sb_info *sbi)
2756	{
2757	return le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
2758	}
2759
2760	static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
2761	{
2762	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2763	void *tmp_ptr = &ckpt->sit_nat_version_bitmap;
2764	int offset;
2765
2766	if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) {
2767	offset = (flag == SIT_BITMAP) ?
2768	le32_to_cpu(ckpt->nat_ver_bitmap_bytesize) : 0;
2769	/*
2770	* if large_nat_bitmap feature is enabled, leave checksum
2771	* protection for all nat/sit bitmaps.
2772	*/
2773	return tmp_ptr + offset + sizeof(__le32);
2774	}
2775
2776	if (__cp_payload(sbi) > 0) {
2777	if (flag == NAT_BITMAP)
2778	return tmp_ptr;
2779	else
2780	return (unsigned char *)ckpt + F2FS_BLKSIZE;
2781	} else {
2782	offset = (flag == NAT_BITMAP) ?
2783	le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
2784	return tmp_ptr + offset;
2785	}
2786	}
2787
2788	static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi)
2789	{
2790	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2791
2792	if (sbi->cur_cp_pack == 2)
2793	start_addr += BLKS_PER_SEG(sbi);
2794	return start_addr;
2795	}
2796
2797	static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi)
2798	{
2799	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2800
2801	if (sbi->cur_cp_pack == 1)
2802	start_addr += BLKS_PER_SEG(sbi);
2803	return start_addr;
2804	}
2805
2806	static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi)
2807	{
2808	sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1;
2809	}
2810
2811	static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi)
2812	{
2813	return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
2814	}
2815
2816	extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
2817	static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
2818	struct inode *inode, bool is_inode)
2819	{
2820	block_t valid_block_count;
2821	unsigned int valid_node_count, avail_user_node_count;
2822	unsigned int avail_user_block_count;
2823	int err;
2824
2825	if (is_inode) {
2826	if (inode) {
2827	err = dquot_alloc_inode(inode);
2828	if (err)
2829	return err;
2830	}
2831	} else {
2832	err = dquot_reserve_block(inode, nr: 1);
2833	if (err)
2834	return err;
2835	}
2836
2837	if (time_to_inject(sbi, FAULT_BLOCK))
2838	goto enospc;
2839
2840	spin_lock(lock: &sbi->stat_lock);
2841
2842	valid_block_count = sbi->total_valid_block_count + 1;
2843	avail_user_block_count = get_available_block_count(sbi, inode,
2844	test_opt(sbi, RESERVE_NODE));
2845
2846	if (unlikely(valid_block_count > avail_user_block_count)) {
2847	spin_unlock(lock: &sbi->stat_lock);
2848	goto enospc;
2849	}
2850
2851	avail_user_node_count = sbi->total_node_count - F2FS_RESERVED_NODE_NUM;
2852	if (test_opt(sbi, RESERVE_NODE) &&
2853	!__allow_reserved_root(sbi, inode, cap: true))
2854	avail_user_node_count -= F2FS_OPTION(sbi).root_reserved_nodes;
2855	valid_node_count = sbi->total_valid_node_count + 1;
2856	if (unlikely(valid_node_count > avail_user_node_count)) {
2857	spin_unlock(lock: &sbi->stat_lock);
2858	goto enospc;
2859	}
2860
2861	sbi->total_valid_node_count++;
2862	sbi->total_valid_block_count++;
2863	spin_unlock(lock: &sbi->stat_lock);
2864
2865	if (inode) {
2866	if (is_inode)
2867	f2fs_mark_inode_dirty_sync(inode, sync: true);
2868	else
2869	f2fs_i_blocks_write(inode, 1, true, true);
2870	}
2871
2872	percpu_counter_inc(fbc: &sbi->alloc_valid_block_count);
2873	return 0;
2874
2875	enospc:
2876	if (is_inode) {
2877	if (inode)
2878	dquot_free_inode(inode);
2879	} else {
2880	dquot_release_reservation_block(inode, nr: 1);
2881	}
2882	return -ENOSPC;
2883	}
2884
2885	static inline void dec_valid_node_count(struct f2fs_sb_info *sbi,
2886	struct inode *inode, bool is_inode)
2887	{
2888	spin_lock(lock: &sbi->stat_lock);
2889
2890	if (unlikely(!sbi->total_valid_block_count ||
2891	!sbi->total_valid_node_count)) {
2892	f2fs_warn(sbi, "dec_valid_node_count: inconsistent block counts, total_valid_block:%u, total_valid_node:%u",
2893	sbi->total_valid_block_count,
2894	sbi->total_valid_node_count);
2895	set_sbi_flag(sbi, type: SBI_NEED_FSCK);
2896	} else {
2897	sbi->total_valid_block_count--;
2898	sbi->total_valid_node_count--;
2899	}
2900
2901	if (sbi->reserved_blocks &&
2902	sbi->current_reserved_blocks < sbi->reserved_blocks)
2903	sbi->current_reserved_blocks++;
2904
2905	spin_unlock(lock: &sbi->stat_lock);
2906
2907	if (is_inode) {
2908	dquot_free_inode(inode);
2909	} else {
2910	if (unlikely(inode->i_blocks == 0)) {
2911	f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%lu, iblocks:%llu",
2912	inode->i_ino,
2913	(unsigned long long)inode->i_blocks);
2914	set_sbi_flag(sbi, type: SBI_NEED_FSCK);
2915	return;
2916	}
2917	f2fs_i_blocks_write(inode, 1, false, true);
2918	}
2919	}
2920
2921	static inline unsigned int valid_node_count(struct f2fs_sb_info *sbi)
2922	{
2923	return sbi->total_valid_node_count;
2924	}
2925
2926	static inline void inc_valid_inode_count(struct f2fs_sb_info *sbi)
2927	{
2928	percpu_counter_inc(fbc: &sbi->total_valid_inode_count);
2929	}
2930
2931	static inline void dec_valid_inode_count(struct f2fs_sb_info *sbi)
2932	{
2933	percpu_counter_dec(fbc: &sbi->total_valid_inode_count);
2934	}
2935
2936	static inline s64 valid_inode_count(struct f2fs_sb_info *sbi)
2937	{
2938	return percpu_counter_sum_positive(fbc: &sbi->total_valid_inode_count);
2939	}
2940
2941	static inline struct folio *f2fs_grab_cache_folio(struct address_space *mapping,
2942	pgoff_t index, bool for_write)
2943	{
2944	struct folio *folio;
2945	unsigned int flags;
2946
2947	if (IS_ENABLED(CONFIG_F2FS_FAULT_INJECTION)) {
2948	fgf_t fgf_flags;
2949
2950	if (!for_write)
2951	fgf_flags = FGP_LOCK | FGP_ACCESSED;
2952	else
2953	fgf_flags = FGP_LOCK;
2954	folio = __filemap_get_folio(mapping, index, fgf_flags, gfp: 0);
2955	if (!IS_ERR(ptr: folio))
2956	return folio;
2957
2958	if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC))
2959	return ERR_PTR(error: -ENOMEM);
2960	}
2961
2962	if (!for_write)
2963	return filemap_grab_folio(mapping, index);
2964
2965	flags = memalloc_nofs_save();
2966	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
2967	gfp: mapping_gfp_mask(mapping));
2968	memalloc_nofs_restore(flags);
2969
2970	return folio;
2971	}
2972
2973	static inline struct folio *f2fs_filemap_get_folio(
2974	struct address_space *mapping, pgoff_t index,
2975	fgf_t fgp_flags, gfp_t gfp_mask)
2976	{
2977	if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET))
2978	return ERR_PTR(error: -ENOMEM);
2979
2980	return __filemap_get_folio(mapping, index, fgf_flags: fgp_flags, gfp: gfp_mask);
2981	}
2982
2983	static inline void f2fs_folio_put(struct folio *folio, bool unlock)
2984	{
2985	if (IS_ERR_OR_NULL(ptr: folio))
2986	return;
2987
2988	if (unlock) {
2989	f2fs_bug_on(F2FS_F_SB(folio), !folio_test_locked(folio));
2990	folio_unlock(folio);
2991	}
2992	folio_put(folio);
2993	}
2994
2995	static inline void f2fs_put_page(struct page *page, bool unlock)
2996	{
2997	if (!page)
2998	return;
2999	f2fs_folio_put(page_folio(page), unlock);
3000	}
3001
3002	static inline void f2fs_put_dnode(struct dnode_of_data *dn)
3003	{
3004	if (dn->node_folio)
3005	f2fs_folio_put(folio: dn->node_folio, unlock: true);
3006	if (dn->inode_folio && dn->node_folio != dn->inode_folio)
3007	f2fs_folio_put(folio: dn->inode_folio, unlock: false);
3008	dn->node_folio = NULL;
3009	dn->inode_folio = NULL;
3010	}
3011
3012	static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name,
3013	size_t size)
3014	{
3015	return kmem_cache_create(name, size, 0, SLAB_RECLAIM_ACCOUNT, NULL);
3016	}
3017
3018	static inline void *f2fs_kmem_cache_alloc_nofail(struct kmem_cache *cachep,
3019	gfp_t flags)
3020	{
3021	void *entry;
3022
3023	entry = kmem_cache_alloc(cachep, flags);
3024	if (!entry)
3025	entry = kmem_cache_alloc(cachep, flags | __GFP_NOFAIL);
3026	return entry;
3027	}
3028
3029	static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep,
3030	gfp_t flags, bool nofail, struct f2fs_sb_info *sbi)
3031	{
3032	if (nofail)
3033	return f2fs_kmem_cache_alloc_nofail(cachep, flags);
3034
3035	if (time_to_inject(sbi, FAULT_SLAB_ALLOC))
3036	return NULL;
3037
3038	return kmem_cache_alloc(cachep, flags);
3039	}
3040
3041	static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
3042	{
3043	if (get_pages(sbi, count_type: F2FS_RD_DATA) || get_pages(sbi, count_type: F2FS_RD_NODE) ||
3044	get_pages(sbi, count_type: F2FS_RD_META) || get_pages(sbi, count_type: F2FS_WB_DATA) ||
3045	get_pages(sbi, count_type: F2FS_WB_CP_DATA) ||
3046	get_pages(sbi, count_type: F2FS_DIO_READ) ||
3047	get_pages(sbi, count_type: F2FS_DIO_WRITE))
3048	return true;
3049
3050	if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
3051	atomic_read(v: &SM_I(sbi)->dcc_info->queued_discard))
3052	return true;
3053
3054	if (SM_I(sbi) && SM_I(sbi)->fcc_info &&
3055	atomic_read(v: &SM_I(sbi)->fcc_info->queued_flush))
3056	return true;
3057	return false;
3058	}
3059
3060	static inline bool is_inflight_read_io(struct f2fs_sb_info *sbi)
3061	{
3062	return get_pages(sbi, count_type: F2FS_RD_DATA) || get_pages(sbi, count_type: F2FS_DIO_READ);
3063	}
3064
3065	static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
3066	{
3067	bool zoned_gc = (type == GC_TIME &&
3068	F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_BLKZONED));
3069
3070	if (sbi->gc_mode == GC_URGENT_HIGH)
3071	return true;
3072
3073	if (sbi->bggc_io_aware == AWARE_READ_IO && is_inflight_read_io(sbi))
3074	return false;
3075	if (sbi->bggc_io_aware == AWARE_ALL_IO && is_inflight_io(sbi, type))
3076	return false;
3077
3078	if (sbi->gc_mode == GC_URGENT_MID)
3079	return true;
3080
3081	if (sbi->gc_mode == GC_URGENT_LOW &&
3082	(type == DISCARD_TIME || type == GC_TIME))
3083	return true;
3084
3085	if (zoned_gc)
3086	return true;
3087
3088	return f2fs_time_over(sbi, type);
3089	}
3090
3091	static inline void f2fs_radix_tree_insert(struct radix_tree_root *root,
3092	unsigned long index, void *item)
3093	{
3094	while (radix_tree_insert(root, index, item))
3095	cond_resched();
3096	}
3097
3098	#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino)
3099
3100	static inline bool IS_INODE(const struct folio *folio)
3101	{
3102	struct f2fs_node *p = F2FS_NODE(folio);
3103
3104	return RAW_IS_INODE(p);
3105	}
3106
3107	static inline int offset_in_addr(struct f2fs_inode *i)
3108	{
3109	return (i->i_inline & F2FS_EXTRA_ATTR) ?
3110	(le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0;
3111	}
3112
3113	static inline __le32 *blkaddr_in_node(struct f2fs_node *node)
3114	{
3115	return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr;
3116	}
3117
3118	static inline int f2fs_has_extra_attr(struct inode *inode);
3119	static inline unsigned int get_dnode_base(struct inode *inode,
3120	struct folio *node_folio)
3121	{
3122	if (!IS_INODE(folio: node_folio))
3123	return 0;
3124
3125	return inode ? get_extra_isize(inode) :
3126	offset_in_addr(i: &F2FS_NODE(folio: node_folio)->i);
3127	}
3128
3129	static inline __le32 *get_dnode_addr(struct inode *inode,
3130	struct folio *node_folio)
3131	{
3132	return blkaddr_in_node(node: F2FS_NODE(folio: node_folio)) +
3133	get_dnode_base(inode, node_folio);
3134	}
3135
3136	static inline block_t data_blkaddr(struct inode *inode,
3137	struct folio *node_folio, unsigned int offset)
3138	{
3139	return le32_to_cpu(*(get_dnode_addr(inode, node_folio) + offset));
3140	}
3141
3142	static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn)
3143	{
3144	return data_blkaddr(inode: dn->inode, node_folio: dn->node_folio, offset: dn->ofs_in_node);
3145	}
3146
3147	static inline int f2fs_test_bit(unsigned int nr, char *addr)
3148	{
3149	int mask;
3150
3151	addr += (nr >> 3);
3152	mask = BIT(7 - (nr & 0x07));
3153	return mask & *addr;
3154	}
3155
3156	static inline void f2fs_set_bit(unsigned int nr, char *addr)
3157	{
3158	int mask;
3159
3160	addr += (nr >> 3);
3161	mask = BIT(7 - (nr & 0x07));
3162	*addr |= mask;
3163	}
3164
3165	static inline void f2fs_clear_bit(unsigned int nr, char *addr)
3166	{
3167	int mask;
3168
3169	addr += (nr >> 3);
3170	mask = BIT(7 - (nr & 0x07));
3171	*addr &= ~mask;
3172	}
3173
3174	static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr)
3175	{
3176	int mask;
3177	int ret;
3178
3179	addr += (nr >> 3);
3180	mask = BIT(7 - (nr & 0x07));
3181	ret = mask & *addr;
3182	*addr |= mask;
3183	return ret;
3184	}
3185
3186	static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr)
3187	{
3188	int mask;
3189	int ret;
3190
3191	addr += (nr >> 3);
3192	mask = BIT(7 - (nr & 0x07));
3193	ret = mask & *addr;
3194	*addr &= ~mask;
3195	return ret;
3196	}
3197
3198	static inline void f2fs_change_bit(unsigned int nr, char *addr)
3199	{
3200	int mask;
3201
3202	addr += (nr >> 3);
3203	mask = BIT(7 - (nr & 0x07));
3204	*addr ^= mask;
3205	}
3206
3207	/*
3208	* On-disk inode flags (f2fs_inode::i_flags)
3209	*/
3210	#define F2FS_COMPR_FL 0x00000004 /* Compress file */
3211	#define F2FS_SYNC_FL 0x00000008 /* Synchronous updates */
3212	#define F2FS_IMMUTABLE_FL 0x00000010 /* Immutable file */
3213	#define F2FS_APPEND_FL 0x00000020 /* writes to file may only append */
3214	#define F2FS_NODUMP_FL 0x00000040 /* do not dump file */
3215	#define F2FS_NOATIME_FL 0x00000080 /* do not update atime */
3216	#define F2FS_NOCOMP_FL 0x00000400 /* Don't compress */
3217	#define F2FS_INDEX_FL 0x00001000 /* hash-indexed directory */
3218	#define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */
3219	#define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */
3220	#define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */
3221	#define F2FS_DEVICE_ALIAS_FL 0x80000000 /* File for aliasing a device */
3222
3223	#define F2FS_QUOTA_DEFAULT_FL (F2FS_NOATIME_FL | F2FS_IMMUTABLE_FL)
3224
3225	/* Flags that should be inherited by new inodes from their parent. */
3226	#define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \
3227	F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
3228	F2FS_CASEFOLD_FL)
3229
3230	/* Flags that are appropriate for regular files (all but dir-specific ones). */
3231	#define F2FS_REG_FLMASK (~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
3232	F2FS_CASEFOLD_FL))
3233
3234	/* Flags that are appropriate for non-directories/regular files. */
3235	#define F2FS_OTHER_FLMASK (F2FS_NODUMP_FL | F2FS_NOATIME_FL)
3236
3237	#define IS_DEVICE_ALIASING(inode) (F2FS_I(inode)->i_flags & F2FS_DEVICE_ALIAS_FL)
3238
3239	static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags)
3240	{
3241	if (S_ISDIR(mode))
3242	return flags;
3243	else if (S_ISREG(mode))
3244	return flags & F2FS_REG_FLMASK;
3245	else
3246	return flags & F2FS_OTHER_FLMASK;
3247	}
3248
3249	static inline void __mark_inode_dirty_flag(struct inode *inode,
3250	int flag, bool set)
3251	{
3252	switch (flag) {
3253	case FI_INLINE_XATTR:
3254	case FI_INLINE_DATA:
3255	case FI_INLINE_DENTRY:
3256	case FI_NEW_INODE:
3257	if (set)
3258	return;
3259	fallthrough;
3260	case FI_DATA_EXIST:
3261	case FI_PIN_FILE:
3262	case FI_COMPRESS_RELEASED:
3263	f2fs_mark_inode_dirty_sync(inode, sync: true);
3264	}
3265	}
3266
3267	static inline void set_inode_flag(struct inode *inode, int flag)
3268	{
3269	set_bit(nr: flag, addr: F2FS_I(inode)->flags);
3270	__mark_inode_dirty_flag(inode, flag, set: true);
3271	}
3272
3273	static inline int is_inode_flag_set(struct inode *inode, int flag)
3274	{
3275	return test_bit(flag, F2FS_I(inode)->flags);
3276	}
3277
3278	static inline void clear_inode_flag(struct inode *inode, int flag)
3279	{
3280	clear_bit(nr: flag, addr: F2FS_I(inode)->flags);
3281	__mark_inode_dirty_flag(inode, flag, set: false);
3282	}
3283
3284	static inline bool f2fs_verity_in_progress(struct inode *inode)
3285	{
3286	return IS_ENABLED(CONFIG_FS_VERITY) &&
3287	is_inode_flag_set(inode, flag: FI_VERITY_IN_PROGRESS);
3288	}
3289
3290	static inline void set_acl_inode(struct inode *inode, umode_t mode)
3291	{
3292	F2FS_I(inode)->i_acl_mode = mode;
3293	set_inode_flag(inode, flag: FI_ACL_MODE);
3294	f2fs_mark_inode_dirty_sync(inode, sync: false);
3295	}
3296
3297	static inline void f2fs_i_links_write(struct inode *inode, bool inc)
3298	{
3299	if (inc)
3300	inc_nlink(inode);
3301	else
3302	drop_nlink(inode);
3303	f2fs_mark_inode_dirty_sync(inode, sync: true);
3304	}
3305
3306	static inline void f2fs_i_blocks_write(struct inode *inode,
3307	block_t diff, bool add, bool claim)
3308	{
3309	bool clean = !is_inode_flag_set(inode, flag: FI_DIRTY_INODE);
3310	bool recover = is_inode_flag_set(inode, flag: FI_AUTO_RECOVER);
3311
3312	/* add = 1, claim = 1 should be dquot_reserve_block in pair */
3313	if (add) {
3314	if (claim)
3315	dquot_claim_block(inode, nr: diff);
3316	else
3317	dquot_alloc_block_nofail(inode, nr: diff);
3318	} else {
3319	dquot_free_block(inode, nr: diff);
3320	}
3321
3322	f2fs_mark_inode_dirty_sync(inode, sync: true);
3323	if (clean || recover)
3324	set_inode_flag(inode, flag: FI_AUTO_RECOVER);
3325	}
3326
3327	static inline bool f2fs_is_atomic_file(struct inode *inode);
3328
3329	static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size)
3330	{
3331	bool clean = !is_inode_flag_set(inode, flag: FI_DIRTY_INODE);
3332	bool recover = is_inode_flag_set(inode, flag: FI_AUTO_RECOVER);
3333
3334	if (i_size_read(inode) == i_size)
3335	return;
3336
3337	i_size_write(inode, i_size);
3338
3339	if (f2fs_is_atomic_file(inode))
3340	return;
3341
3342	f2fs_mark_inode_dirty_sync(inode, sync: true);
3343	if (clean || recover)
3344	set_inode_flag(inode, flag: FI_AUTO_RECOVER);
3345	}
3346
3347	static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth)
3348	{
3349	F2FS_I(inode)->i_current_depth = depth;
3350	f2fs_mark_inode_dirty_sync(inode, sync: true);
3351	}
3352
3353	static inline void f2fs_i_gc_failures_write(struct inode *inode,
3354	unsigned int count)
3355	{
3356	F2FS_I(inode)->i_gc_failures = count;
3357	f2fs_mark_inode_dirty_sync(inode, sync: true);
3358	}
3359
3360	static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid)
3361	{
3362	F2FS_I(inode)->i_xattr_nid = xnid;
3363	f2fs_mark_inode_dirty_sync(inode, sync: true);
3364	}
3365
3366	static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino)
3367	{
3368	F2FS_I(inode)->i_pino = pino;
3369	f2fs_mark_inode_dirty_sync(inode, sync: true);
3370	}
3371
3372	static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri)
3373	{
3374	struct f2fs_inode_info *fi = F2FS_I(inode);
3375
3376	if (ri->i_inline & F2FS_INLINE_XATTR)
3377	set_bit(nr: FI_INLINE_XATTR, addr: fi->flags);
3378	if (ri->i_inline & F2FS_INLINE_DATA)
3379	set_bit(nr: FI_INLINE_DATA, addr: fi->flags);
3380	if (ri->i_inline & F2FS_INLINE_DENTRY)
3381	set_bit(nr: FI_INLINE_DENTRY, addr: fi->flags);
3382	if (ri->i_inline & F2FS_DATA_EXIST)
3383	set_bit(nr: FI_DATA_EXIST, addr: fi->flags);
3384	if (ri->i_inline & F2FS_EXTRA_ATTR)
3385	set_bit(nr: FI_EXTRA_ATTR, addr: fi->flags);
3386	if (ri->i_inline & F2FS_PIN_FILE)
3387	set_bit(nr: FI_PIN_FILE, addr: fi->flags);
3388	if (ri->i_inline & F2FS_COMPRESS_RELEASED)
3389	set_bit(nr: FI_COMPRESS_RELEASED, addr: fi->flags);
3390	}
3391
3392	static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri)
3393	{
3394	ri->i_inline = 0;
3395
3396	if (is_inode_flag_set(inode, flag: FI_INLINE_XATTR))
3397	ri->i_inline |= F2FS_INLINE_XATTR;
3398	if (is_inode_flag_set(inode, flag: FI_INLINE_DATA))
3399	ri->i_inline |= F2FS_INLINE_DATA;
3400	if (is_inode_flag_set(inode, flag: FI_INLINE_DENTRY))
3401	ri->i_inline |= F2FS_INLINE_DENTRY;
3402	if (is_inode_flag_set(inode, flag: FI_DATA_EXIST))
3403	ri->i_inline |= F2FS_DATA_EXIST;
3404	if (is_inode_flag_set(inode, flag: FI_EXTRA_ATTR))
3405	ri->i_inline |= F2FS_EXTRA_ATTR;
3406	if (is_inode_flag_set(inode, flag: FI_PIN_FILE))
3407	ri->i_inline |= F2FS_PIN_FILE;
3408	if (is_inode_flag_set(inode, flag: FI_COMPRESS_RELEASED))
3409	ri->i_inline |= F2FS_COMPRESS_RELEASED;
3410	}
3411
3412	static inline int f2fs_has_extra_attr(struct inode *inode)
3413	{
3414	return is_inode_flag_set(inode, flag: FI_EXTRA_ATTR);
3415	}
3416
3417	static inline int f2fs_has_inline_xattr(struct inode *inode)
3418	{
3419	return is_inode_flag_set(inode, flag: FI_INLINE_XATTR);
3420	}
3421
3422	static inline int f2fs_compressed_file(struct inode *inode)
3423	{
3424	return S_ISREG(inode->i_mode) &&
3425	is_inode_flag_set(inode, flag: FI_COMPRESSED_FILE);
3426	}
3427
3428	static inline bool f2fs_need_compress_data(struct inode *inode)
3429	{
3430	int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode;
3431
3432	if (!f2fs_compressed_file(inode))
3433	return false;
3434
3435	if (compress_mode == COMPR_MODE_FS)
3436	return true;
3437	else if (compress_mode == COMPR_MODE_USER &&
3438	is_inode_flag_set(inode, flag: FI_ENABLE_COMPRESS))
3439	return true;
3440
3441	return false;
3442	}
3443
3444	static inline unsigned int addrs_per_page(struct inode *inode,
3445	bool is_inode)
3446	{
3447	unsigned int addrs = is_inode ? (CUR_ADDRS_PER_INODE(inode) -
3448	get_inline_xattr_addrs(inode)) : DEF_ADDRS_PER_BLOCK;
3449
3450	if (f2fs_compressed_file(inode))
3451	return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size);
3452	return addrs;
3453	}
3454
3455	static inline
3456	void *inline_xattr_addr(struct inode *inode, const struct folio *folio)
3457	{
3458	struct f2fs_inode *ri = F2FS_INODE(folio);
3459
3460	return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE -
3461	get_inline_xattr_addrs(inode)]);
3462	}
3463
3464	static inline int inline_xattr_size(struct inode *inode)
3465	{
3466	if (f2fs_has_inline_xattr(inode))
3467	return get_inline_xattr_addrs(inode) * sizeof(__le32);
3468	return 0;
3469	}
3470
3471	/*
3472	* Notice: check inline_data flag without inode page lock is unsafe.
3473	* It could change at any time by f2fs_convert_inline_folio().
3474	*/
3475	static inline int f2fs_has_inline_data(struct inode *inode)
3476	{
3477	return is_inode_flag_set(inode, flag: FI_INLINE_DATA);
3478	}
3479
3480	static inline int f2fs_exist_data(struct inode *inode)
3481	{
3482	return is_inode_flag_set(inode, flag: FI_DATA_EXIST);
3483	}
3484
3485	static inline int f2fs_is_mmap_file(struct inode *inode)
3486	{
3487	return is_inode_flag_set(inode, flag: FI_MMAP_FILE);
3488	}
3489
3490	static inline bool f2fs_is_pinned_file(struct inode *inode)
3491	{
3492	return is_inode_flag_set(inode, flag: FI_PIN_FILE);
3493	}
3494
3495	static inline bool f2fs_is_atomic_file(struct inode *inode)
3496	{
3497	return is_inode_flag_set(inode, flag: FI_ATOMIC_FILE);
3498	}
3499
3500	static inline bool f2fs_is_cow_file(struct inode *inode)
3501	{
3502	return is_inode_flag_set(inode, flag: FI_COW_FILE);
3503	}
3504
3505	static inline void *inline_data_addr(struct inode *inode, struct folio *folio)
3506	{
3507	__le32 *addr = get_dnode_addr(inode, node_folio: folio);
3508
3509	return (void *)(addr + DEF_INLINE_RESERVED_SIZE);
3510	}
3511
3512	static inline int f2fs_has_inline_dentry(struct inode *inode)
3513	{
3514	return is_inode_flag_set(inode, flag: FI_INLINE_DENTRY);
3515	}
3516
3517	static inline int is_file(struct inode *inode, int type)
3518	{
3519	return F2FS_I(inode)->i_advise & type;
3520	}
3521
3522	static inline void set_file(struct inode *inode, int type)
3523	{
3524	if (is_file(inode, type))
3525	return;
3526	F2FS_I(inode)->i_advise |= type;
3527	f2fs_mark_inode_dirty_sync(inode, sync: true);
3528	}
3529
3530	static inline void clear_file(struct inode *inode, int type)
3531	{
3532	if (!is_file(inode, type))
3533	return;
3534	F2FS_I(inode)->i_advise &= ~type;
3535	f2fs_mark_inode_dirty_sync(inode, sync: true);
3536	}
3537
3538	static inline bool f2fs_is_time_consistent(struct inode *inode)
3539	{
3540	struct timespec64 ts = inode_get_atime(inode);
3541
3542	if (!timespec64_equal(a: F2FS_I(inode)->i_disk_time, b: &ts))
3543	return false;
3544	ts = inode_get_ctime(inode);
3545	if (!timespec64_equal(a: F2FS_I(inode)->i_disk_time + 1, b: &ts))
3546	return false;
3547	ts = inode_get_mtime(inode);
3548	if (!timespec64_equal(a: F2FS_I(inode)->i_disk_time + 2, b: &ts))
3549	return false;
3550	return true;
3551	}
3552
3553	static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
3554	{
3555	bool ret;
3556
3557	if (dsync) {
3558	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3559
3560	spin_lock(lock: &sbi->inode_lock[DIRTY_META]);
3561	ret = list_empty(head: &F2FS_I(inode)->gdirty_list);
3562	spin_unlock(lock: &sbi->inode_lock[DIRTY_META]);
3563	return ret;
3564	}
3565	if (!is_inode_flag_set(inode, flag: FI_AUTO_RECOVER) ||
3566	file_keep_isize(inode) ||
3567	i_size_read(inode) & ~PAGE_MASK)
3568	return false;
3569
3570	if (!f2fs_is_time_consistent(inode))
3571	return false;
3572
3573	spin_lock(lock: &F2FS_I(inode)->i_size_lock);
3574	ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
3575	spin_unlock(lock: &F2FS_I(inode)->i_size_lock);
3576
3577	return ret;
3578	}
3579
3580	static inline bool f2fs_readonly(struct super_block *sb)
3581	{
3582	return sb_rdonly(sb);
3583	}
3584
3585	static inline bool f2fs_cp_error(struct f2fs_sb_info *sbi)
3586	{
3587	return is_set_ckpt_flags(sbi, CP_ERROR_FLAG);
3588	}
3589
3590	static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi,
3591	size_t size, gfp_t flags)
3592	{
3593	if (time_to_inject(sbi, FAULT_KMALLOC))
3594	return NULL;
3595
3596	return kmalloc(size, flags);
3597	}
3598
3599	static inline void *f2fs_getname(struct f2fs_sb_info *sbi)
3600	{
3601	if (time_to_inject(sbi, FAULT_KMALLOC))
3602	return NULL;
3603
3604	return __getname();
3605	}
3606
3607	static inline void f2fs_putname(char *buf)
3608	{
3609	__putname(buf);
3610	}
3611
3612	static inline void *f2fs_kzalloc(struct f2fs_sb_info *sbi,
3613	size_t size, gfp_t flags)
3614	{
3615	return f2fs_kmalloc(sbi, size, flags: flags | __GFP_ZERO);
3616	}
3617
3618	static inline void *f2fs_kvmalloc(struct f2fs_sb_info *sbi,
3619	size_t size, gfp_t flags)
3620	{
3621	if (time_to_inject(sbi, FAULT_KVMALLOC))
3622	return NULL;
3623
3624	return kvmalloc(size, flags);
3625	}
3626
3627	static inline void *f2fs_kvzalloc(struct f2fs_sb_info *sbi,
3628	size_t size, gfp_t flags)
3629	{
3630	return f2fs_kvmalloc(sbi, size, flags: flags | __GFP_ZERO);
3631	}
3632
3633	static inline void *f2fs_vmalloc(struct f2fs_sb_info *sbi, size_t size)
3634	{
3635	if (time_to_inject(sbi, FAULT_VMALLOC))
3636	return NULL;
3637
3638	return vmalloc(size);
3639	}
3640
3641	static inline int get_extra_isize(struct inode *inode)
3642	{
3643	return F2FS_I(inode)->i_extra_isize / sizeof(__le32);
3644	}
3645
3646	static inline int get_inline_xattr_addrs(struct inode *inode)
3647	{
3648	return F2FS_I(inode)->i_inline_xattr_size;
3649	}
3650
3651	#define f2fs_get_inode_mode(i) \
3652	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
3653	(F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
3654
3655	#define F2FS_MIN_EXTRA_ATTR_SIZE (sizeof(__le32))
3656
3657	#define F2FS_TOTAL_EXTRA_ATTR_SIZE \
3658	(offsetof(struct f2fs_inode, i_extra_end) - \
3659	offsetof(struct f2fs_inode, i_extra_isize)) \
3660
3661	#define F2FS_OLD_ATTRIBUTE_SIZE (offsetof(struct f2fs_inode, i_addr))
3662	#define F2FS_FITS_IN_INODE(f2fs_inode, extra_isize, field) \
3663	((offsetof(typeof(*(f2fs_inode)), field) + \
3664	sizeof((f2fs_inode)->field)) \
3665	<= (F2FS_OLD_ATTRIBUTE_SIZE + (extra_isize))) \
3666
3667	#define __is_large_section(sbi) (SEGS_PER_SEC(sbi) > 1)
3668
3669	#define __is_meta_io(fio) (PAGE_TYPE_OF_BIO((fio)->type) == META)
3670
3671	bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3672	block_t blkaddr, int type);
3673	static inline void verify_blkaddr(struct f2fs_sb_info *sbi,
3674	block_t blkaddr, int type)
3675	{
3676	if (!f2fs_is_valid_blkaddr(sbi, blkaddr, type))
3677	f2fs_err(sbi, "invalid blkaddr: %u, type: %d, run fsck to fix.",
3678	blkaddr, type);
3679	}
3680
3681	static inline bool __is_valid_data_blkaddr(block_t blkaddr)
3682	{
3683	if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR ||
3684	blkaddr == COMPRESS_ADDR)
3685	return false;
3686	return true;
3687	}
3688
3689	/*
3690	* file.c
3691	*/
3692	int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync);
3693	int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock);
3694	int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock);
3695	int f2fs_truncate(struct inode *inode);
3696	int f2fs_getattr(struct mnt_idmap *idmap, const struct path *path,
3697	struct kstat *stat, u32 request_mask, unsigned int flags);
3698	int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
3699	struct iattr *attr);
3700	int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end);
3701	void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count);
3702	int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag,
3703	bool readonly, bool need_lock);
3704	int f2fs_precache_extents(struct inode *inode);
3705	int f2fs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
3706	int f2fs_fileattr_set(struct mnt_idmap *idmap,
3707	struct dentry *dentry, struct file_kattr *fa);
3708	long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
3709	long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
3710	int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid);
3711	int f2fs_pin_file_control(struct inode *inode, bool inc);
3712
3713	/*
3714	* inode.c
3715	*/
3716	void f2fs_set_inode_flags(struct inode *inode);
3717	bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio);
3718	void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct folio *folio);
3719	struct inode *f2fs_iget(struct super_block *sb, unsigned long ino);
3720	struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino);
3721	int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink);
3722	void f2fs_update_inode(struct inode *inode, struct folio *node_folio);
3723	void f2fs_update_inode_page(struct inode *inode);
3724	int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc);
3725	void f2fs_remove_donate_inode(struct inode *inode);
3726	void f2fs_evict_inode(struct inode *inode);
3727	void f2fs_handle_failed_inode(struct inode *inode);
3728
3729	/*
3730	* namei.c
3731	*/
3732	int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
3733	bool hot, bool set);
3734	struct dentry *f2fs_get_parent(struct dentry *child);
3735	int f2fs_get_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
3736	struct inode **new_inode);
3737
3738	/*
3739	* dir.c
3740	*/
3741	#if IS_ENABLED(CONFIG_UNICODE)
3742	int f2fs_init_casefolded_name(const struct inode *dir,
3743	struct f2fs_filename *fname);
3744	void f2fs_free_casefolded_name(struct f2fs_filename *fname);
3745	#else
3746	static inline int f2fs_init_casefolded_name(const struct inode *dir,
3747	struct f2fs_filename *fname)
3748	{
3749	return 0;
3750	}
3751
3752	static inline void f2fs_free_casefolded_name(struct f2fs_filename *fname)
3753	{
3754	}
3755	#endif /* CONFIG_UNICODE */
3756
3757	int f2fs_setup_filename(struct inode *dir, const struct qstr *iname,
3758	int lookup, struct f2fs_filename *fname);
3759	int f2fs_prepare_lookup(struct inode *dir, struct dentry *dentry,
3760	struct f2fs_filename *fname);
3761	void f2fs_free_filename(struct f2fs_filename *fname);
3762	struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d,
3763	const struct f2fs_filename *fname, int *max_slots,
3764	bool use_hash);
3765	int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
3766	unsigned int start_pos, struct fscrypt_str *fstr);
3767	void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
3768	struct f2fs_dentry_ptr *d);
3769	struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
3770	const struct f2fs_filename *fname, struct folio *dfolio);
3771	void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode,
3772	unsigned int current_depth);
3773	int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots);
3774	void f2fs_drop_nlink(struct inode *dir, struct inode *inode);
3775	struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
3776	const struct f2fs_filename *fname, struct folio **res_folio);
3777	struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
3778	const struct qstr *child, struct folio **res_folio);
3779	struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct folio **f);
3780	ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr,
3781	struct folio **folio);
3782	void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
3783	struct folio *folio, struct inode *inode);
3784	bool f2fs_has_enough_room(struct inode *dir, struct folio *ifolio,
3785	const struct f2fs_filename *fname);
3786	void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d,
3787	const struct fscrypt_str *name, f2fs_hash_t name_hash,
3788	unsigned int bit_pos);
3789	int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
3790	struct inode *inode, nid_t ino, umode_t mode);
3791	int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname,
3792	struct inode *inode, nid_t ino, umode_t mode);
3793	int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
3794	struct inode *inode, nid_t ino, umode_t mode);
3795	void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio,
3796	struct inode *dir, struct inode *inode);
3797	int f2fs_do_tmpfile(struct inode *inode, struct inode *dir,
3798	struct f2fs_filename *fname);
3799	bool f2fs_empty_dir(struct inode *dir);
3800
3801	static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode)
3802	{
3803	if (fscrypt_is_nokey_name(dentry))
3804	return -ENOKEY;
3805	return f2fs_do_add_link(dir: d_inode(dentry: dentry->d_parent), name: &dentry->d_name,
3806	inode, ino: inode->i_ino, mode: inode->i_mode);
3807	}
3808
3809	/*
3810	* super.c
3811	*/
3812	int f2fs_inode_dirtied(struct inode *inode, bool sync);
3813	void f2fs_inode_synced(struct inode *inode);
3814	int f2fs_dquot_initialize(struct inode *inode);
3815	int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
3816	int f2fs_do_quota_sync(struct super_block *sb, int type);
3817	loff_t max_file_blocks(struct inode *inode);
3818	void f2fs_quota_off_umount(struct super_block *sb);
3819	void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag);
3820	void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason);
3821	void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error);
3822	int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
3823	int f2fs_sync_fs(struct super_block *sb, int sync);
3824	int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
3825
3826	/*
3827	* hash.c
3828	*/
3829	void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname);
3830
3831	/*
3832	* node.c
3833	*/
3834	struct node_info;
3835	enum node_type;
3836
3837	int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
3838	bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type);
3839	bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct folio *folio);
3840	void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi);
3841	void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio);
3842	void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi);
3843	int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid);
3844	bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
3845	bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino);
3846	int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
3847	struct node_info *ni, bool checkpoint_context);
3848	pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
3849	int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
3850	int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
3851	int f2fs_truncate_xattr_node(struct inode *inode);
3852	int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
3853	unsigned int seq_id);
3854	int f2fs_remove_inode_page(struct inode *inode);
3855	struct folio *f2fs_new_inode_folio(struct inode *inode);
3856	struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs);
3857	void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid);
3858	struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid,
3859	enum node_type node_type);
3860	struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino);
3861	struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid);
3862	int f2fs_move_node_folio(struct folio *node_folio, int gc_type);
3863	void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
3864	int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
3865	struct writeback_control *wbc, bool atomic,
3866	unsigned int *seq_id);
3867	int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
3868	struct writeback_control *wbc,
3869	bool do_balance, enum iostat_type io_type);
3870	int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount);
3871	bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid);
3872	void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid);
3873	void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid);
3874	int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink);
3875	int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio);
3876	int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio);
3877	int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio);
3878	int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
3879	unsigned int segno, struct f2fs_summary_block *sum);
3880	int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3881	int f2fs_build_node_manager(struct f2fs_sb_info *sbi);
3882	void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi);
3883	int __init f2fs_create_node_manager_caches(void);
3884	void f2fs_destroy_node_manager_caches(void);
3885
3886	/*
3887	* segment.c
3888	*/
3889	bool f2fs_need_SSR(struct f2fs_sb_info *sbi);
3890	int f2fs_commit_atomic_write(struct inode *inode);
3891	void f2fs_abort_atomic_write(struct inode *inode, bool clean);
3892	void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need);
3893	void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg);
3894	int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino);
3895	int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi);
3896	int f2fs_flush_device_cache(struct f2fs_sb_info *sbi);
3897	void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free);
3898	void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr,
3899	unsigned int len);
3900	bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr);
3901	int f2fs_start_discard_thread(struct f2fs_sb_info *sbi);
3902	void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi);
3903	void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi);
3904	bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi);
3905	void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
3906	struct cp_control *cpc);
3907	void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi);
3908	block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi);
3909	int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
3910	void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
3911	int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
3912	bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno);
3913	int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
3914	int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi);
3915	void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi);
3916	void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi);
3917	int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3918	unsigned int start, unsigned int end);
3919	int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force);
3920	int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi);
3921	int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
3922	int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
3923	bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3924	struct cp_control *cpc);
3925	struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno);
3926	void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src,
3927	block_t blk_addr);
3928	void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio,
3929	enum iostat_type io_type);
3930	void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio);
3931	void f2fs_outplace_write_data(struct dnode_of_data *dn,
3932	struct f2fs_io_info *fio);
3933	int f2fs_inplace_write_data(struct f2fs_io_info *fio);
3934	void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3935	block_t old_blkaddr, block_t new_blkaddr,
3936	bool recover_curseg, bool recover_newaddr,
3937	bool from_gc);
3938	void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3939	block_t old_addr, block_t new_addr,
3940	unsigned char version, bool recover_curseg,
3941	bool recover_newaddr);
3942	enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi,
3943	enum log_type seg_type);
3944	int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio,
3945	block_t old_blkaddr, block_t *new_blkaddr,
3946	struct f2fs_summary *sum, int type,
3947	struct f2fs_io_info *fio);
3948	void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3949	block_t blkaddr, unsigned int blkcnt);
3950	void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type,
3951	bool ordered, bool locked);
3952	#define f2fs_wait_on_page_writeback(page, type, ordered, locked) \
3953	f2fs_folio_wait_writeback(page_folio(page), type, ordered, locked)
3954	void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
3955	void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3956	block_t len);
3957	void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3958	void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3959	int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3960	unsigned int val, int alloc);
3961	void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3962	int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi);
3963	int f2fs_build_segment_manager(struct f2fs_sb_info *sbi);
3964	void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi);
3965	int __init f2fs_create_segment_manager_caches(void);
3966	void f2fs_destroy_segment_manager_caches(void);
3967	int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint);
3968	enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3969	enum page_type type, enum temp_type temp);
3970	unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi);
3971	unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
3972	unsigned int segno);
3973	unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi,
3974	unsigned int segno);
3975
3976	static inline struct inode *fio_inode(struct f2fs_io_info *fio)
3977	{
3978	return fio->folio->mapping->host;
3979	}
3980
3981	#define DEF_FRAGMENT_SIZE 4
3982	#define MIN_FRAGMENT_SIZE 1
3983	#define MAX_FRAGMENT_SIZE 512
3984
3985	static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi)
3986	{
3987	return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG ||
3988	F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK;
3989	}
3990
3991	/*
3992	* checkpoint.c
3993	*/
3994	void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
3995	unsigned char reason);
3996	void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi);
3997	struct folio *f2fs_grab_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index);
3998	struct folio *f2fs_get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index);
3999	struct folio *f2fs_get_meta_folio_retry(struct f2fs_sb_info *sbi, pgoff_t index);
4000	struct folio *f2fs_get_tmp_folio(struct f2fs_sb_info *sbi, pgoff_t index);
4001	bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
4002	block_t blkaddr, int type);
4003	bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi,
4004	block_t blkaddr, int type);
4005	int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
4006	int type, bool sync);
4007	void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index,
4008	unsigned int ra_blocks);
4009	long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
4010	long nr_to_write, enum iostat_type io_type);
4011	void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
4012	void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
4013	void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all);
4014	bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode);
4015	void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
4016	unsigned int devidx, int type);
4017	bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
4018	unsigned int devidx, int type);
4019	int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi);
4020	void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi);
4021	void f2fs_add_orphan_inode(struct inode *inode);
4022	void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino);
4023	int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi);
4024	int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
4025	void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio);
4026	void f2fs_remove_dirty_inode(struct inode *inode);
4027	int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
4028	bool from_cp);
4029	void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
4030	u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
4031	int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
4032	void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
4033	int __init f2fs_create_checkpoint_caches(void);
4034	void f2fs_destroy_checkpoint_caches(void);
4035	int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi);
4036	int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi);
4037	void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi);
4038	void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi);
4039
4040	/*
4041	* data.c
4042	*/
4043	int __init f2fs_init_bioset(void);
4044	void f2fs_destroy_bioset(void);
4045	bool f2fs_is_cp_guaranteed(const struct folio *folio);
4046	int f2fs_init_bio_entry_cache(void);
4047	void f2fs_destroy_bio_entry_cache(void);
4048	void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
4049	enum page_type type);
4050	int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi);
4051	void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
4052	void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
4053	struct inode *inode, struct folio *folio,
4054	nid_t ino, enum page_type type);
4055	void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
4056	struct bio **bio, struct folio *folio);
4057	void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
4058	int f2fs_submit_page_bio(struct f2fs_io_info *fio);
4059	int f2fs_merge_page_bio(struct f2fs_io_info *fio);
4060	void f2fs_submit_page_write(struct f2fs_io_info *fio);
4061	struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
4062	block_t blk_addr, sector_t *sector);
4063	int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr);
4064	void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr);
4065	void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr);
4066	int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count);
4067	int f2fs_reserve_new_block(struct dnode_of_data *dn);
4068	int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index);
4069	int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
4070	struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index,
4071	blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs);
4072	struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index,
4073	pgoff_t *next_pgofs);
4074	struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index,
4075	bool for_write);
4076	struct folio *f2fs_get_new_data_folio(struct inode *inode,
4077	struct folio *ifolio, pgoff_t index, bool new_i_size);
4078	int f2fs_do_write_data_page(struct f2fs_io_info *fio);
4079	int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag);
4080	int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4081	u64 start, u64 len);
4082	int f2fs_encrypt_one_page(struct f2fs_io_info *fio);
4083	bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio);
4084	bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio);
4085	int f2fs_write_single_data_page(struct folio *folio, int *submitted,
4086	struct bio **bio, sector_t *last_block,
4087	struct writeback_control *wbc,
4088	enum iostat_type io_type,
4089	int compr_blocks, bool allow_balance);
4090	void f2fs_write_failed(struct inode *inode, loff_t to);
4091	void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length);
4092	bool f2fs_release_folio(struct folio *folio, gfp_t wait);
4093	bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
4094	void f2fs_clear_page_cache_dirty_tag(struct folio *folio);
4095	int f2fs_init_post_read_processing(void);
4096	void f2fs_destroy_post_read_processing(void);
4097	int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi);
4098	void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi);
4099	extern const struct iomap_ops f2fs_iomap_ops;
4100
4101	/*
4102	* gc.c
4103	*/
4104	int f2fs_start_gc_thread(struct f2fs_sb_info *sbi);
4105	void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi);
4106	block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode);
4107	int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control);
4108	void f2fs_build_gc_manager(struct f2fs_sb_info *sbi);
4109	int f2fs_gc_range(struct f2fs_sb_info *sbi,
4110	unsigned int start_seg, unsigned int end_seg,
4111	bool dry_run, unsigned int dry_run_sections);
4112	int f2fs_resize_fs(struct file *filp, __u64 block_count);
4113	int __init f2fs_create_garbage_collection_cache(void);
4114	void f2fs_destroy_garbage_collection_cache(void);
4115	/* victim selection function for cleaning and SSR */
4116	int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result,
4117	int gc_type, int type, char alloc_mode,
4118	unsigned long long age, bool one_time);
4119
4120	/*
4121	* recovery.c
4122	*/
4123	int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only);
4124	bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi);
4125	int __init f2fs_create_recovery_cache(void);
4126	void f2fs_destroy_recovery_cache(void);
4127
4128	/*
4129	* debug.c
4130	*/
4131	#ifdef CONFIG_F2FS_STAT_FS
4132	enum {
4133	DEVSTAT_INUSE,
4134	DEVSTAT_DIRTY,
4135	DEVSTAT_FULL,
4136	DEVSTAT_FREE,
4137	DEVSTAT_PREFREE,
4138	DEVSTAT_MAX,
4139	};
4140
4141	struct f2fs_dev_stats {
4142	unsigned int devstats[2][DEVSTAT_MAX]; /* 0: segs, 1: secs */
4143	};
4144
4145	struct f2fs_stat_info {
4146	struct list_head stat_list;
4147	struct f2fs_sb_info *sbi;
4148	int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs;
4149	int main_area_segs, main_area_sections, main_area_zones;
4150	unsigned long long hit_cached[NR_EXTENT_CACHES];
4151	unsigned long long hit_rbtree[NR_EXTENT_CACHES];
4152	unsigned long long total_ext[NR_EXTENT_CACHES];
4153	unsigned long long hit_total[NR_EXTENT_CACHES];
4154	int ext_tree[NR_EXTENT_CACHES];
4155	int zombie_tree[NR_EXTENT_CACHES];
4156	int ext_node[NR_EXTENT_CACHES];
4157	/* to count memory footprint */
4158	unsigned long long ext_mem[NR_EXTENT_CACHES];
4159	/* for read extent cache */
4160	unsigned long long hit_largest;
4161	/* for block age extent cache */
4162	unsigned long long allocated_data_blocks;
4163	int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta;
4164	int ndirty_data, ndirty_qdata;
4165	unsigned int ndirty_dirs, ndirty_files, ndirty_all;
4166	unsigned int nquota_files, ndonate_files;
4167	int nats, dirty_nats, sits, dirty_sits;
4168	int free_nids, avail_nids, alloc_nids;
4169	int total_count, utilization;
4170	int nr_wb_cp_data, nr_wb_data;
4171	int nr_rd_data, nr_rd_node, nr_rd_meta;
4172	int nr_dio_read, nr_dio_write;
4173	unsigned int io_skip_bggc, other_skip_bggc;
4174	int nr_flushing, nr_flushed, flush_list_empty;
4175	int nr_discarding, nr_discarded;
4176	int nr_discard_cmd;
4177	unsigned int undiscard_blks;
4178	int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
4179	unsigned int cur_ckpt_time, peak_ckpt_time;
4180	int inline_xattr, inline_inode, inline_dir, append, update, orphans;
4181	int compr_inode, swapfile_inode;
4182	unsigned long long compr_blocks;
4183	int aw_cnt, max_aw_cnt;
4184	unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
4185	unsigned int bimodal, avg_vblocks;
4186	int util_free, util_valid, util_invalid;
4187	int rsvd_segs, overp_segs;
4188	int dirty_count, node_pages, meta_pages, compress_pages;
4189	int compress_page_hit;
4190	int prefree_count, free_segs, free_secs;
4191	int cp_call_count[MAX_CALL_TYPE], cp_count;
4192	int gc_call_count[MAX_CALL_TYPE];
4193	int gc_segs[2][2];
4194	int gc_secs[2][2];
4195	int tot_blks, data_blks, node_blks;
4196	int bg_data_blks, bg_node_blks;
4197	int blkoff[NR_CURSEG_TYPE];
4198	int curseg[NR_CURSEG_TYPE];
4199	int cursec[NR_CURSEG_TYPE];
4200	int curzone[NR_CURSEG_TYPE];
4201	unsigned int dirty_seg[NR_CURSEG_TYPE];
4202	unsigned int full_seg[NR_CURSEG_TYPE];
4203	unsigned int valid_blks[NR_CURSEG_TYPE];
4204
4205	unsigned int meta_count[META_MAX];
4206	unsigned int segment_count[2];
4207	unsigned int block_count[2];
4208	unsigned int inplace_count;
4209	unsigned long long base_mem, cache_mem, page_mem;
4210	struct f2fs_dev_stats *dev_stats;
4211	};
4212
4213	static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi)
4214	{
4215	return (struct f2fs_stat_info *)sbi->stat_info;
4216	}
4217
4218	#define stat_inc_cp_call_count(sbi, foreground) \
4219	atomic_inc(&sbi->cp_call_count[(foreground)])
4220	#define stat_inc_cp_count(sbi) (F2FS_STAT(sbi)->cp_count++)
4221	#define stat_io_skip_bggc_count(sbi) ((sbi)->io_skip_bggc++)
4222	#define stat_other_skip_bggc_count(sbi) ((sbi)->other_skip_bggc++)
4223	#define stat_inc_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]++)
4224	#define stat_dec_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]--)
4225	#define stat_inc_total_hit(sbi, type) (atomic64_inc(&(sbi)->total_hit_ext[type]))
4226	#define stat_inc_rbtree_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_rbtree[type]))
4227	#define stat_inc_largest_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_largest))
4228	#define stat_inc_cached_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_cached[type]))
4229	#define stat_inc_inline_xattr(inode) \
4230	do { \
4231	if (f2fs_has_inline_xattr(inode)) \
4232	(atomic_inc(&F2FS_I_SB(inode)->inline_xattr)); \
4233	} while (0)
4234	#define stat_dec_inline_xattr(inode) \
4235	do { \
4236	if (f2fs_has_inline_xattr(inode)) \
4237	(atomic_dec(&F2FS_I_SB(inode)->inline_xattr)); \
4238	} while (0)
4239	#define stat_inc_inline_inode(inode) \
4240	do { \
4241	if (f2fs_has_inline_data(inode)) \
4242	(atomic_inc(&F2FS_I_SB(inode)->inline_inode)); \
4243	} while (0)
4244	#define stat_dec_inline_inode(inode) \
4245	do { \
4246	if (f2fs_has_inline_data(inode)) \
4247	(atomic_dec(&F2FS_I_SB(inode)->inline_inode)); \
4248	} while (0)
4249	#define stat_inc_inline_dir(inode) \
4250	do { \
4251	if (f2fs_has_inline_dentry(inode)) \
4252	(atomic_inc(&F2FS_I_SB(inode)->inline_dir)); \
4253	} while (0)
4254	#define stat_dec_inline_dir(inode) \
4255	do { \
4256	if (f2fs_has_inline_dentry(inode)) \
4257	(atomic_dec(&F2FS_I_SB(inode)->inline_dir)); \
4258	} while (0)
4259	#define stat_inc_compr_inode(inode) \
4260	do { \
4261	if (f2fs_compressed_file(inode)) \
4262	(atomic_inc(&F2FS_I_SB(inode)->compr_inode)); \
4263	} while (0)
4264	#define stat_dec_compr_inode(inode) \
4265	do { \
4266	if (f2fs_compressed_file(inode)) \
4267	(atomic_dec(&F2FS_I_SB(inode)->compr_inode)); \
4268	} while (0)
4269	#define stat_add_compr_blocks(inode, blocks) \
4270	(atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
4271	#define stat_sub_compr_blocks(inode, blocks) \
4272	(atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
4273	#define stat_inc_swapfile_inode(inode) \
4274	(atomic_inc(&F2FS_I_SB(inode)->swapfile_inode))
4275	#define stat_dec_swapfile_inode(inode) \
4276	(atomic_dec(&F2FS_I_SB(inode)->swapfile_inode))
4277	#define stat_inc_atomic_inode(inode) \
4278	(atomic_inc(&F2FS_I_SB(inode)->atomic_files))
4279	#define stat_dec_atomic_inode(inode) \
4280	(atomic_dec(&F2FS_I_SB(inode)->atomic_files))
4281	#define stat_inc_meta_count(sbi, blkaddr) \
4282	do { \
4283	if (blkaddr < SIT_I(sbi)->sit_base_addr) \
4284	atomic_inc(&(sbi)->meta_count[META_CP]); \
4285	else if (blkaddr < NM_I(sbi)->nat_blkaddr) \
4286	atomic_inc(&(sbi)->meta_count[META_SIT]); \
4287	else if (blkaddr < SM_I(sbi)->ssa_blkaddr) \
4288	atomic_inc(&(sbi)->meta_count[META_NAT]); \
4289	else if (blkaddr < SM_I(sbi)->main_blkaddr) \
4290	atomic_inc(&(sbi)->meta_count[META_SSA]); \
4291	} while (0)
4292	#define stat_inc_seg_type(sbi, curseg) \
4293	((sbi)->segment_count[(curseg)->alloc_type]++)
4294	#define stat_inc_block_count(sbi, curseg) \
4295	((sbi)->block_count[(curseg)->alloc_type]++)
4296	#define stat_inc_inplace_blocks(sbi) \
4297	(atomic_inc(&(sbi)->inplace_count))
4298	#define stat_update_max_atomic_write(inode) \
4299	do { \
4300	int cur = atomic_read(&F2FS_I_SB(inode)->atomic_files); \
4301	int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt); \
4302	if (cur > max) \
4303	atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur); \
4304	} while (0)
4305	#define stat_inc_gc_call_count(sbi, foreground) \
4306	(F2FS_STAT(sbi)->gc_call_count[(foreground)]++)
4307	#define stat_inc_gc_sec_count(sbi, type, gc_type) \
4308	(F2FS_STAT(sbi)->gc_secs[(type)][(gc_type)]++)
4309	#define stat_inc_gc_seg_count(sbi, type, gc_type) \
4310	(F2FS_STAT(sbi)->gc_segs[(type)][(gc_type)]++)
4311
4312	#define stat_inc_tot_blk_count(si, blks) \
4313	((si)->tot_blks += (blks))
4314
4315	#define stat_inc_data_blk_count(sbi, blks, gc_type) \
4316	do { \
4317	struct f2fs_stat_info *si = F2FS_STAT(sbi); \
4318	stat_inc_tot_blk_count(si, blks); \
4319	si->data_blks += (blks); \
4320	si->bg_data_blks += ((gc_type) == BG_GC) ? (blks) : 0; \
4321	} while (0)
4322
4323	#define stat_inc_node_blk_count(sbi, blks, gc_type) \
4324	do { \
4325	struct f2fs_stat_info *si = F2FS_STAT(sbi); \
4326	stat_inc_tot_blk_count(si, blks); \
4327	si->node_blks += (blks); \
4328	si->bg_node_blks += ((gc_type) == BG_GC) ? (blks) : 0; \
4329	} while (0)
4330
4331	int f2fs_build_stats(struct f2fs_sb_info *sbi);
4332	void f2fs_destroy_stats(struct f2fs_sb_info *sbi);
4333	void __init f2fs_create_root_stats(void);
4334	void f2fs_destroy_root_stats(void);
4335	void f2fs_update_sit_info(struct f2fs_sb_info *sbi);
4336	#else
4337	#define stat_inc_cp_call_count(sbi, foreground) do { } while (0)
4338	#define stat_inc_cp_count(sbi) do { } while (0)
4339	#define stat_io_skip_bggc_count(sbi) do { } while (0)
4340	#define stat_other_skip_bggc_count(sbi) do { } while (0)
4341	#define stat_inc_dirty_inode(sbi, type) do { } while (0)
4342	#define stat_dec_dirty_inode(sbi, type) do { } while (0)
4343	#define stat_inc_total_hit(sbi, type) do { } while (0)
4344	#define stat_inc_rbtree_node_hit(sbi, type) do { } while (0)
4345	#define stat_inc_largest_node_hit(sbi) do { } while (0)
4346	#define stat_inc_cached_node_hit(sbi, type) do { } while (0)
4347	#define stat_inc_inline_xattr(inode) do { } while (0)
4348	#define stat_dec_inline_xattr(inode) do { } while (0)
4349	#define stat_inc_inline_inode(inode) do { } while (0)
4350	#define stat_dec_inline_inode(inode) do { } while (0)
4351	#define stat_inc_inline_dir(inode) do { } while (0)
4352	#define stat_dec_inline_dir(inode) do { } while (0)
4353	#define stat_inc_compr_inode(inode) do { } while (0)
4354	#define stat_dec_compr_inode(inode) do { } while (0)
4355	#define stat_add_compr_blocks(inode, blocks) do { } while (0)
4356	#define stat_sub_compr_blocks(inode, blocks) do { } while (0)
4357	#define stat_inc_swapfile_inode(inode) do { } while (0)
4358	#define stat_dec_swapfile_inode(inode) do { } while (0)
4359	#define stat_inc_atomic_inode(inode) do { } while (0)
4360	#define stat_dec_atomic_inode(inode) do { } while (0)
4361	#define stat_update_max_atomic_write(inode) do { } while (0)
4362	#define stat_inc_meta_count(sbi, blkaddr) do { } while (0)
4363	#define stat_inc_seg_type(sbi, curseg) do { } while (0)
4364	#define stat_inc_block_count(sbi, curseg) do { } while (0)
4365	#define stat_inc_inplace_blocks(sbi) do { } while (0)
4366	#define stat_inc_gc_call_count(sbi, foreground) do { } while (0)
4367	#define stat_inc_gc_sec_count(sbi, type, gc_type) do { } while (0)
4368	#define stat_inc_gc_seg_count(sbi, type, gc_type) do { } while (0)
4369	#define stat_inc_tot_blk_count(si, blks) do { } while (0)
4370	#define stat_inc_data_blk_count(sbi, blks, gc_type) do { } while (0)
4371	#define stat_inc_node_blk_count(sbi, blks, gc_type) do { } while (0)
4372
4373	static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; }
4374	static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { }
4375	static inline void __init f2fs_create_root_stats(void) { }
4376	static inline void f2fs_destroy_root_stats(void) { }
4377	static inline void f2fs_update_sit_info(struct f2fs_sb_info *sbi) {}
4378	#endif
4379
4380	extern const struct file_operations f2fs_dir_operations;
4381	extern const struct file_operations f2fs_file_operations;
4382	extern const struct inode_operations f2fs_file_inode_operations;
4383	extern const struct address_space_operations f2fs_dblock_aops;
4384	extern const struct address_space_operations f2fs_node_aops;
4385	extern const struct address_space_operations f2fs_meta_aops;
4386	extern const struct inode_operations f2fs_dir_inode_operations;
4387	extern const struct inode_operations f2fs_symlink_inode_operations;
4388	extern const struct inode_operations f2fs_encrypted_symlink_inode_operations;
4389	extern const struct inode_operations f2fs_special_inode_operations;
4390	extern struct kmem_cache *f2fs_inode_entry_slab;
4391
4392	/*
4393	* inline.c
4394	*/
4395	bool f2fs_may_inline_data(struct inode *inode);
4396	bool f2fs_sanity_check_inline_data(struct inode *inode, struct folio *ifolio);
4397	bool f2fs_may_inline_dentry(struct inode *inode);
4398	void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio);
4399	void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio,
4400	u64 from);
4401	int f2fs_read_inline_data(struct inode *inode, struct folio *folio);
4402	int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio);
4403	int f2fs_convert_inline_inode(struct inode *inode);
4404	int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry);
4405	int f2fs_write_inline_data(struct inode *inode, struct folio *folio);
4406	int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio);
4407	struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
4408	const struct f2fs_filename *fname, struct folio **res_folio,
4409	bool use_hash);
4410	int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
4411	struct folio *ifolio);
4412	int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
4413	struct inode *inode, nid_t ino, umode_t mode);
4414	void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
4415	struct folio *folio, struct inode *dir, struct inode *inode);
4416	bool f2fs_empty_inline_dir(struct inode *dir);
4417	int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
4418	struct fscrypt_str *fstr);
4419	int f2fs_inline_data_fiemap(struct inode *inode,
4420	struct fiemap_extent_info *fieinfo,
4421	__u64 start, __u64 len);
4422
4423	/*
4424	* shrinker.c
4425	*/
4426	unsigned long f2fs_shrink_count(struct shrinker *shrink,
4427	struct shrink_control *sc);
4428	unsigned long f2fs_shrink_scan(struct shrinker *shrink,
4429	struct shrink_control *sc);
4430	unsigned int f2fs_donate_files(void);
4431	void f2fs_reclaim_caches(unsigned int reclaim_caches_kb);
4432	void f2fs_join_shrinker(struct f2fs_sb_info *sbi);
4433	void f2fs_leave_shrinker(struct f2fs_sb_info *sbi);
4434
4435	/*
4436	* extent_cache.c
4437	*/
4438	bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio);
4439	void f2fs_init_extent_tree(struct inode *inode);
4440	void f2fs_drop_extent_tree(struct inode *inode);
4441	void f2fs_destroy_extent_node(struct inode *inode);
4442	void f2fs_destroy_extent_tree(struct inode *inode);
4443	void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi);
4444	int __init f2fs_create_extent_cache(void);
4445	void f2fs_destroy_extent_cache(void);
4446
4447	/* read extent cache ops */
4448	void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio);
4449	bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs,
4450	struct extent_info *ei);
4451	bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index,
4452	block_t *blkaddr);
4453	void f2fs_update_read_extent_cache(struct dnode_of_data *dn);
4454	void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn,
4455	pgoff_t fofs, block_t blkaddr, unsigned int len);
4456	unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi,
4457	int nr_shrink);
4458
4459	/* block age extent cache ops */
4460	void f2fs_init_age_extent_tree(struct inode *inode);
4461	bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs,
4462	struct extent_info *ei);
4463	void f2fs_update_age_extent_cache(struct dnode_of_data *dn);
4464	void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn,
4465	pgoff_t fofs, unsigned int len);
4466	unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi,
4467	int nr_shrink);
4468
4469	/*
4470	* sysfs.c
4471	*/
4472	#define MIN_RA_MUL 2
4473	#define MAX_RA_MUL 256
4474
4475	int __init f2fs_init_sysfs(void);
4476	void f2fs_exit_sysfs(void);
4477	int f2fs_register_sysfs(struct f2fs_sb_info *sbi);
4478	void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi);
4479
4480	/* verity.c */
4481	extern const struct fsverity_operations f2fs_verityops;
4482
4483	/*
4484	* crypto support
4485	*/
4486	static inline bool f2fs_encrypted_file(struct inode *inode)
4487	{
4488	return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
4489	}
4490
4491	static inline void f2fs_set_encrypted_inode(struct inode *inode)
4492	{
4493	#ifdef CONFIG_FS_ENCRYPTION
4494	file_set_encrypt(inode);
4495	f2fs_set_inode_flags(inode);
4496	#endif
4497	}
4498
4499	/*
4500	* Returns true if the reads of the inode's data need to undergo some
4501	* postprocessing step, like decryption or authenticity verification.
4502	*/
4503	static inline bool f2fs_post_read_required(struct inode *inode)
4504	{
4505	return f2fs_encrypted_file(inode) || fsverity_active(inode) ||
4506	f2fs_compressed_file(inode);
4507	}
4508
4509	static inline bool f2fs_used_in_atomic_write(struct inode *inode)
4510	{
4511	return f2fs_is_atomic_file(inode) || f2fs_is_cow_file(inode);
4512	}
4513
4514	static inline bool f2fs_meta_inode_gc_required(struct inode *inode)
4515	{
4516	return f2fs_post_read_required(inode) || f2fs_used_in_atomic_write(inode);
4517	}
4518
4519	/*
4520	* compress.c
4521	*/
4522	#ifdef CONFIG_F2FS_FS_COMPRESSION
4523	enum cluster_check_type {
4524	CLUSTER_IS_COMPR, /* check only if compressed cluster */
4525	CLUSTER_COMPR_BLKS, /* return # of compressed blocks in a cluster */
4526	CLUSTER_RAW_BLKS /* return # of raw blocks in a cluster */
4527	};
4528	bool f2fs_is_compressed_page(struct folio *folio);
4529	struct folio *f2fs_compress_control_folio(struct folio *folio);
4530	int f2fs_prepare_compress_overwrite(struct inode *inode,
4531	struct page **pagep, pgoff_t index, void **fsdata);
4532	bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
4533	pgoff_t index, unsigned copied);
4534	int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock);
4535	void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio);
4536	bool f2fs_is_compress_backend_ready(struct inode *inode);
4537	bool f2fs_is_compress_level_valid(int alg, int lvl);
4538	int __init f2fs_init_compress_mempool(void);
4539	void f2fs_destroy_compress_mempool(void);
4540	void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task);
4541	void f2fs_end_read_compressed_page(struct folio *folio, bool failed,
4542	block_t blkaddr, bool in_task);
4543	bool f2fs_cluster_is_empty(struct compress_ctx *cc);
4544	bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
4545	bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages,
4546	int index, int nr_pages, bool uptodate);
4547	bool f2fs_sanity_check_cluster(struct dnode_of_data *dn);
4548	void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio);
4549	int f2fs_write_multi_pages(struct compress_ctx *cc,
4550	int *submitted,
4551	struct writeback_control *wbc,
4552	enum iostat_type io_type);
4553	int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index);
4554	bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index);
4555	void f2fs_update_read_extent_tree_range_compressed(struct inode *inode,
4556	pgoff_t fofs, block_t blkaddr,
4557	unsigned int llen, unsigned int c_len);
4558	int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
4559	unsigned nr_pages, sector_t *last_block_in_bio,
4560	struct readahead_control *rac, bool for_write);
4561	struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc);
4562	void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
4563	bool in_task);
4564	void f2fs_put_folio_dic(struct folio *folio, bool in_task);
4565	unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn,
4566	unsigned int ofs_in_node);
4567	int f2fs_init_compress_ctx(struct compress_ctx *cc);
4568	void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
4569	void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
4570	int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
4571	void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
4572	int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
4573	void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
4574	int __init f2fs_init_compress_cache(void);
4575	void f2fs_destroy_compress_cache(void);
4576	struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi);
4577	void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi,
4578	block_t blkaddr, unsigned int len);
4579	bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio,
4580	block_t blkaddr);
4581	void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino);
4582	#define inc_compr_inode_stat(inode) \
4583	do { \
4584	struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
4585	sbi->compr_new_inode++; \
4586	} while (0)
4587	#define add_compr_block_stat(inode, blocks) \
4588	do { \
4589	struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
4590	int diff = F2FS_I(inode)->i_cluster_size - blocks; \
4591	sbi->compr_written_block += blocks; \
4592	sbi->compr_saved_block += diff; \
4593	} while (0)
4594	#else
4595	static inline bool f2fs_is_compressed_page(struct folio *folio) { return false; }
4596	static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
4597	{
4598	if (!f2fs_compressed_file(inode))
4599	return true;
4600	/* not support compression */
4601	return false;
4602	}
4603	static inline bool f2fs_is_compress_level_valid(int alg, int lvl) { return false; }
4604	static inline struct folio *f2fs_compress_control_folio(struct folio *folio)
4605	{
4606	WARN_ON_ONCE(1);
4607	return ERR_PTR(-EINVAL);
4608	}
4609	static inline int __init f2fs_init_compress_mempool(void) { return 0; }
4610	static inline void f2fs_destroy_compress_mempool(void) { }
4611	static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic,
4612	bool in_task) { }
4613	static inline void f2fs_end_read_compressed_page(struct folio *folio,
4614	bool failed, block_t blkaddr, bool in_task)
4615	{
4616	WARN_ON_ONCE(1);
4617	}
4618	static inline void f2fs_put_folio_dic(struct folio *folio, bool in_task)
4619	{
4620	WARN_ON_ONCE(1);
4621	}
4622	static inline unsigned int f2fs_cluster_blocks_are_contiguous(
4623	struct dnode_of_data *dn, unsigned int ofs_in_node) { return 0; }
4624	static inline bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) { return false; }
4625	static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
4626	static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
4627	static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
4628	static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
4629	static inline int __init f2fs_init_compress_cache(void) { return 0; }
4630	static inline void f2fs_destroy_compress_cache(void) { }
4631	static inline void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi,
4632	block_t blkaddr, unsigned int len) { }
4633	static inline bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi,
4634	struct folio *folio, block_t blkaddr) { return false; }
4635	static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi,
4636	nid_t ino) { }
4637	#define inc_compr_inode_stat(inode) do { } while (0)
4638	static inline int f2fs_is_compressed_cluster(
4639	struct inode *inode,
4640	pgoff_t index) { return 0; }
4641	static inline bool f2fs_is_sparse_cluster(
4642	struct inode *inode,
4643	pgoff_t index) { return true; }
4644	static inline void f2fs_update_read_extent_tree_range_compressed(
4645	struct inode *inode,
4646	pgoff_t fofs, block_t blkaddr,
4647	unsigned int llen, unsigned int c_len) { }
4648	#endif
4649
4650	static inline int set_compress_context(struct inode *inode)
4651	{
4652	#ifdef CONFIG_F2FS_FS_COMPRESSION
4653	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4654	struct f2fs_inode_info *fi = F2FS_I(inode);
4655
4656	fi->i_compress_algorithm = F2FS_OPTION(sbi).compress_algorithm;
4657	fi->i_log_cluster_size = F2FS_OPTION(sbi).compress_log_size;
4658	fi->i_compress_flag = F2FS_OPTION(sbi).compress_chksum ?
4659	BIT(COMPRESS_CHKSUM) : 0;
4660	fi->i_cluster_size = BIT(fi->i_log_cluster_size);
4661	if ((fi->i_compress_algorithm == COMPRESS_LZ4 ||
4662	fi->i_compress_algorithm == COMPRESS_ZSTD) &&
4663	F2FS_OPTION(sbi).compress_level)
4664	fi->i_compress_level = F2FS_OPTION(sbi).compress_level;
4665	fi->i_flags |= F2FS_COMPR_FL;
4666	set_inode_flag(inode, flag: FI_COMPRESSED_FILE);
4667	stat_inc_compr_inode(inode);
4668	inc_compr_inode_stat(inode);
4669	f2fs_mark_inode_dirty_sync(inode, sync: true);
4670	return 0;
4671	#else
4672	return -EOPNOTSUPP;
4673	#endif
4674	}
4675
4676	static inline bool f2fs_disable_compressed_file(struct inode *inode)
4677	{
4678	struct f2fs_inode_info *fi = F2FS_I(inode);
4679
4680	f2fs_down_write(sem: &fi->i_sem);
4681
4682	if (!f2fs_compressed_file(inode)) {
4683	f2fs_up_write(sem: &fi->i_sem);
4684	return true;
4685	}
4686	if (f2fs_is_mmap_file(inode) || atomic_read(v: &fi->writeback) ||
4687	(S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))) {
4688	f2fs_up_write(sem: &fi->i_sem);
4689	return false;
4690	}
4691
4692	fi->i_flags &= ~F2FS_COMPR_FL;
4693	stat_dec_compr_inode(inode);
4694	clear_inode_flag(inode, flag: FI_COMPRESSED_FILE);
4695	f2fs_mark_inode_dirty_sync(inode, sync: true);
4696
4697	f2fs_up_write(sem: &fi->i_sem);
4698	return true;
4699	}
4700
4701	#define F2FS_FEATURE_FUNCS(name, flagname) \
4702	static inline bool f2fs_sb_has_##name(struct f2fs_sb_info *sbi) \
4703	{ \
4704	return F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_##flagname); \
4705	}
4706
4707	F2FS_FEATURE_FUNCS(encrypt, ENCRYPT);
4708	F2FS_FEATURE_FUNCS(blkzoned, BLKZONED);
4709	F2FS_FEATURE_FUNCS(extra_attr, EXTRA_ATTR);
4710	F2FS_FEATURE_FUNCS(project_quota, PRJQUOTA);
4711	F2FS_FEATURE_FUNCS(inode_chksum, INODE_CHKSUM);
4712	F2FS_FEATURE_FUNCS(flexible_inline_xattr, FLEXIBLE_INLINE_XATTR);
4713	F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO);
4714	F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME);
4715	F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND);
4716	F2FS_FEATURE_FUNCS(verity, VERITY);
4717	F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
4718	F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
4719	F2FS_FEATURE_FUNCS(compression, COMPRESSION);
4720	F2FS_FEATURE_FUNCS(readonly, RO);
4721	F2FS_FEATURE_FUNCS(device_alias, DEVICE_ALIAS);
4722	F2FS_FEATURE_FUNCS(packed_ssa, PACKED_SSA);
4723
4724	#ifdef CONFIG_BLK_DEV_ZONED
4725	static inline bool f2fs_zone_is_seq(struct f2fs_sb_info *sbi, int devi,
4726	unsigned int zone)
4727	{
4728	return test_bit(zone, FDEV(devi).blkz_seq);
4729	}
4730
4731	static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
4732	block_t blkaddr)
4733	{
4734	return f2fs_zone_is_seq(sbi, devi, zone: blkaddr / sbi->blocks_per_blkz);
4735	}
4736	#endif
4737
4738	static inline int f2fs_bdev_index(struct f2fs_sb_info *sbi,
4739	struct block_device *bdev)
4740	{
4741	int i;
4742
4743	if (!f2fs_is_multi_device(sbi))
4744	return 0;
4745
4746	for (i = 0; i < sbi->s_ndevs; i++)
4747	if (FDEV(i).bdev == bdev)
4748	return i;
4749
4750	WARN_ON(1);
4751	return -1;
4752	}
4753
4754	static inline bool f2fs_hw_should_discard(struct f2fs_sb_info *sbi)
4755	{
4756	return f2fs_sb_has_blkzoned(sbi);
4757	}
4758
4759	static inline bool f2fs_bdev_support_discard(struct block_device *bdev)
4760	{
4761	return bdev_max_discard_sectors(bdev) || bdev_is_zoned(bdev);
4762	}
4763
4764	static inline bool f2fs_hw_support_discard(struct f2fs_sb_info *sbi)
4765	{
4766	int i;
4767
4768	if (!f2fs_is_multi_device(sbi))
4769	return f2fs_bdev_support_discard(bdev: sbi->sb->s_bdev);
4770
4771	for (i = 0; i < sbi->s_ndevs; i++)
4772	if (f2fs_bdev_support_discard(FDEV(i).bdev))
4773	return true;
4774	return false;
4775	}
4776
4777	static inline unsigned int f2fs_hw_discard_granularity(struct f2fs_sb_info *sbi)
4778	{
4779	int i = 1;
4780	unsigned int discard_granularity = bdev_discard_granularity(bdev: sbi->sb->s_bdev);
4781
4782	if (f2fs_is_multi_device(sbi))
4783	for (; i < sbi->s_ndevs && !bdev_is_zoned(FDEV(i).bdev); i++)
4784	discard_granularity = max_t(unsigned int, discard_granularity,
4785	bdev_discard_granularity(FDEV(i).bdev));
4786	return discard_granularity;
4787	}
4788
4789	static inline bool f2fs_realtime_discard_enable(struct f2fs_sb_info *sbi)
4790	{
4791	return (test_opt(sbi, DISCARD) && f2fs_hw_support_discard(sbi)) ||
4792	f2fs_hw_should_discard(sbi);
4793	}
4794
4795	static inline bool f2fs_hw_is_readonly(struct f2fs_sb_info *sbi)
4796	{
4797	int i;
4798
4799	if (!f2fs_is_multi_device(sbi))
4800	return bdev_read_only(bdev: sbi->sb->s_bdev);
4801
4802	for (i = 0; i < sbi->s_ndevs; i++)
4803	if (bdev_read_only(FDEV(i).bdev))
4804	return true;
4805	return false;
4806	}
4807
4808	static inline bool f2fs_dev_is_readonly(struct f2fs_sb_info *sbi)
4809	{
4810	return f2fs_sb_has_readonly(sbi) || f2fs_hw_is_readonly(sbi);
4811	}
4812
4813	static inline bool f2fs_lfs_mode(struct f2fs_sb_info *sbi)
4814	{
4815	return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
4816	}
4817
4818	static inline bool f2fs_is_sequential_zone_area(struct f2fs_sb_info *sbi,
4819	block_t blkaddr)
4820	{
4821	if (f2fs_sb_has_blkzoned(sbi)) {
4822	#ifdef CONFIG_BLK_DEV_ZONED
4823	int devi = f2fs_target_device_index(sbi, blkaddr);
4824
4825	if (!bdev_is_zoned(FDEV(devi).bdev))
4826	return false;
4827
4828	if (f2fs_is_multi_device(sbi)) {
4829	if (blkaddr < FDEV(devi).start_blk ||
4830	blkaddr > FDEV(devi).end_blk) {
4831	f2fs_err(sbi, "Invalid block %x", blkaddr);
4832	return false;
4833	}
4834	blkaddr -= FDEV(devi).start_blk;
4835	}
4836
4837	return f2fs_blkz_is_seq(sbi, devi, blkaddr);
4838	#else
4839	return false;
4840	#endif
4841	}
4842	return false;
4843	}
4844
4845	static inline bool f2fs_low_mem_mode(struct f2fs_sb_info *sbi)
4846	{
4847	return F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_LOW;
4848	}
4849
4850	static inline bool f2fs_may_compress(struct inode *inode)
4851	{
4852	if (IS_SWAPFILE(inode) || f2fs_is_pinned_file(inode) ||
4853	f2fs_is_atomic_file(inode) || f2fs_has_inline_data(inode) ||
4854	f2fs_is_mmap_file(inode))
4855	return false;
4856	return S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode);
4857	}
4858
4859	static inline void f2fs_i_compr_blocks_update(struct inode *inode,
4860	u64 blocks, bool add)
4861	{
4862	struct f2fs_inode_info *fi = F2FS_I(inode);
4863	int diff = fi->i_cluster_size - blocks;
4864
4865	/* don't update i_compr_blocks if saved blocks were released */
4866	if (!add && !atomic_read(v: &fi->i_compr_blocks))
4867	return;
4868
4869	if (add) {
4870	atomic_add(i: diff, v: &fi->i_compr_blocks);
4871	stat_add_compr_blocks(inode, diff);
4872	} else {
4873	atomic_sub(i: diff, v: &fi->i_compr_blocks);
4874	stat_sub_compr_blocks(inode, diff);
4875	}
4876	f2fs_mark_inode_dirty_sync(inode, sync: true);
4877	}
4878
4879	static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi,
4880	int flag)
4881	{
4882	if (!f2fs_is_multi_device(sbi))
4883	return false;
4884	if (flag != F2FS_GET_BLOCK_DIO)
4885	return false;
4886	return sbi->aligned_blksize;
4887	}
4888
4889	static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
4890	{
4891	return fsverity_active(inode) &&
4892	idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
4893	}
4894
4895	#ifdef CONFIG_F2FS_FAULT_INJECTION
4896	extern int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate,
4897	unsigned long type, enum fault_option fo);
4898	#else
4899	static inline int f2fs_build_fault_attr(struct f2fs_sb_info *sbi,
4900	unsigned long rate, unsigned long type,
4901	enum fault_option fo)
4902	{
4903	return 0;
4904	}
4905	#endif
4906
4907	static inline bool is_journalled_quota(struct f2fs_sb_info *sbi)
4908	{
4909	#ifdef CONFIG_QUOTA
4910	if (f2fs_sb_has_quota_ino(sbi))
4911	return true;
4912	if (F2FS_OPTION(sbi).s_qf_names[USRQUOTA] ||
4913	F2FS_OPTION(sbi).s_qf_names[GRPQUOTA] ||
4914	F2FS_OPTION(sbi).s_qf_names[PRJQUOTA])
4915	return true;
4916	#endif
4917	return false;
4918	}
4919
4920	static inline bool f2fs_block_unit_discard(struct f2fs_sb_info *sbi)
4921	{
4922	return F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK;
4923	}
4924
4925	static inline void __f2fs_schedule_timeout(long timeout, bool io)
4926	{
4927	set_current_state(TASK_UNINTERRUPTIBLE);
4928	if (io)
4929	io_schedule_timeout(timeout);
4930	else
4931	schedule_timeout(timeout);
4932	}
4933
4934	#define f2fs_io_schedule_timeout(timeout) \
4935	__f2fs_schedule_timeout(timeout, true)
4936	#define f2fs_schedule_timeout(timeout) \
4937	__f2fs_schedule_timeout(timeout, false)
4938
4939	static inline void f2fs_io_schedule_timeout_killable(long timeout)
4940	{
4941	while (timeout) {
4942	if (fatal_signal_pending(current))
4943	return;
4944	set_current_state(TASK_UNINTERRUPTIBLE);
4945	io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT);
4946	if (timeout <= DEFAULT_SCHEDULE_TIMEOUT)
4947	return;
4948	timeout -= DEFAULT_SCHEDULE_TIMEOUT;
4949	}
4950	}
4951
4952	static inline void f2fs_handle_page_eio(struct f2fs_sb_info *sbi,
4953	struct folio *folio, enum page_type type)
4954	{
4955	pgoff_t ofs = folio->index;
4956
4957	if (unlikely(f2fs_cp_error(sbi)))
4958	return;
4959
4960	if (ofs == sbi->page_eio_ofs[type]) {
4961	if (sbi->page_eio_cnt[type]++ == MAX_RETRY_PAGE_EIO)
4962	set_ckpt_flags(sbi, CP_ERROR_FLAG);
4963	} else {
4964	sbi->page_eio_ofs[type] = ofs;
4965	sbi->page_eio_cnt[type] = 0;
4966	}
4967	}
4968
4969	static inline bool f2fs_is_readonly(struct f2fs_sb_info *sbi)
4970	{
4971	return f2fs_sb_has_readonly(sbi) || f2fs_readonly(sb: sbi->sb);
4972	}
4973
4974	static inline void f2fs_truncate_meta_inode_pages(struct f2fs_sb_info *sbi,
4975	block_t blkaddr, unsigned int cnt)
4976	{
4977	bool need_submit = false;
4978	int i = 0;
4979
4980	do {
4981	struct folio *folio;
4982
4983	folio = filemap_get_folio(mapping: META_MAPPING(sbi), index: blkaddr + i);
4984	if (!IS_ERR(ptr: folio)) {
4985	if (folio_test_writeback(folio))
4986	need_submit = true;
4987	f2fs_folio_put(folio, unlock: false);
4988	}
4989	} while (++i < cnt && !need_submit);
4990
4991	if (need_submit)
4992	f2fs_submit_merged_write_cond(sbi, inode: sbi->meta_inode,
4993	NULL, ino: 0, type: DATA);
4994
4995	truncate_inode_pages_range(mapping: META_MAPPING(sbi),
4996	F2FS_BLK_TO_BYTES((loff_t)blkaddr),
4997	F2FS_BLK_END_BYTES((loff_t)(blkaddr + cnt - 1)));
4998	}
4999
5000	static inline void f2fs_invalidate_internal_cache(struct f2fs_sb_info *sbi,
5001	block_t blkaddr, unsigned int len)
5002	{
5003	f2fs_truncate_meta_inode_pages(sbi, blkaddr, cnt: len);
5004	f2fs_invalidate_compress_pages_range(sbi, blkaddr, len);
5005	}
5006
5007	#define EFSBADCRC EBADMSG /* Bad CRC detected */
5008	#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */
5009
5010	#endif /* _LINUX_F2FS_H */
5011