// SPDX-License-Identifier: GPL-2.0

/*

* fs/f2fs/node.c

*

* Copyright (c) 2012 Samsung Electronics Co., Ltd.

* http://www.samsung.com/

*/

#include <linux/fs.h>

#include <linux/f2fs_fs.h>

#include <linux/mpage.h>

#include <linux/sched/mm.h>

#include <linux/blkdev.h>

#include <linux/folio_batch.h>

#include <linux/swap.h>

#include "f2fs.h"

#include "node.h"

#include "segment.h"

#include "xattr.h"

#include "iostat.h"

#include <trace/events/f2fs.h>

#define on_f2fs_build_free_nids(nm_i) mutex_is_locked(&(nm_i)->build_lock)

static struct kmem_cache *nat_entry_slab;

static struct kmem_cache *free_nid_slab;

static struct kmem_cache *nat_entry_set_slab;

static struct kmem_cache *fsync_node_entry_slab;

static inline bool is_invalid_nid(struct f2fs_sb_info *sbi, nid_t nid)

{

return nid < F2FS_ROOT_INO(sbi) || nid >= NM_I(sbi)->max_nid;

}

/*

* Check whether the given nid is within node id range.

*/

int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid)

{

if (unlikely(is_invalid_nid(sbi, nid))) {

set_sbi_flag(sbi, SBI_NEED_FSCK);

f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.",

__func__, nid);

f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);

return -EFSCORRUPTED;

}

return 0;

}

bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;

struct sysinfo val;

unsigned long avail_ram;

unsigned long mem_size = 0;

bool res = false;

if (!nm_i)

return true;

si_meminfo(&val);

/* only uses low memory */

avail_ram = val.totalram - val.totalhigh;

/*

* give 25%, 25%, 50%, 50%, 25%, 25% memory for each components respectively

*/

if (type == FREE_NIDS) {

mem_size = (nm_i->nid_cnt[FREE_NID] *

sizeof(struct free_nid)) >> PAGE_SHIFT;

res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);

} else if (type == NAT_ENTRIES) {

mem_size = (nm_i->nat_cnt[TOTAL_NAT] *

sizeof(struct nat_entry)) >> PAGE_SHIFT;

res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);

if (excess_cached_nats(sbi))

res = false;

} else if (type == DIRTY_DENTS) {

if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi))

return false;

mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);

res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);

} else if (type == INO_ENTRIES) {

int i;

for (i = 0; i < MAX_INO_ENTRY; i++)

mem_size += sbi->im[i].ino_num *

sizeof(struct ino_entry);

mem_size >>= PAGE_SHIFT;

res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);

} else if (type == READ_EXTENT_CACHE || type == AGE_EXTENT_CACHE) {

enum extent_type etype = type == READ_EXTENT_CACHE ?

EX_READ : EX_BLOCK_AGE;

struct extent_tree_info *eti = &sbi->extent_tree[etype];

mem_size = (atomic_read(&eti->total_ext_tree) *

sizeof(struct extent_tree) +

atomic_read(&eti->total_ext_node) *

sizeof(struct extent_node)) >> PAGE_SHIFT;

res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);

} else if (type == DISCARD_CACHE) {

mem_size = (atomic_read(&dcc->discard_cmd_cnt) *

sizeof(struct discard_cmd)) >> PAGE_SHIFT;

res = mem_size < (avail_ram * nm_i->ram_thresh / 100);

} else if (type == COMPRESS_PAGE) {

#ifdef CONFIG_F2FS_FS_COMPRESSION

unsigned long free_ram = val.freeram;

/*

* free memory is lower than watermark or cached page count

* exceed threshold, deny caching compress page.

*/

res = (free_ram > avail_ram * sbi->compress_watermark / 100) &&

(COMPRESS_MAPPING(sbi)->nrpages <

free_ram * sbi->compress_percent / 100);

#else

res = false;

#endif

} else {

if (!bdi_wb_dirty_exceeded(sbi->sb->s_bdi))

return true;

}

return res;

}

static void clear_node_folio_dirty(struct folio *folio)

{

if (folio_test_dirty(folio)) {

f2fs_clear_page_cache_dirty_tag(folio);

folio_clear_dirty_for_io(folio);

dec_page_count(F2FS_F_SB(folio), F2FS_DIRTY_NODES);

}

folio_clear_uptodate(folio);

}

static struct folio *get_current_nat_folio(struct f2fs_sb_info *sbi, nid_t nid)

{

return f2fs_get_meta_folio_retry(sbi, current_nat_addr(sbi, nid));

}

static struct folio *get_next_nat_folio(struct f2fs_sb_info *sbi, nid_t nid)

{

struct folio *src_folio;

struct folio *dst_folio;

pgoff_t dst_off;

void *src_addr;

void *dst_addr;

struct f2fs_nm_info *nm_i = NM_I(sbi);

dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid));

/* get current nat block page with lock */

src_folio = get_current_nat_folio(sbi, nid);

if (IS_ERR(src_folio))

return src_folio;

dst_folio = f2fs_grab_meta_folio(sbi, dst_off);

f2fs_bug_on(sbi, folio_test_dirty(src_folio));

src_addr = folio_address(src_folio);

dst_addr = folio_address(dst_folio);

memcpy(dst_addr, src_addr, PAGE_SIZE);

folio_mark_dirty(dst_folio);

f2fs_folio_put(src_folio, true);

set_to_next_nat(nm_i, nid);

return dst_folio;

}

static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi,

nid_t nid, bool no_fail)

{

struct nat_entry *new;

new = f2fs_kmem_cache_alloc(nat_entry_slab,

GFP_F2FS_ZERO, no_fail, sbi);

if (new) {

nat_set_nid(new, nid);

nat_reset_flag(new);

}

return new;

}

static void __free_nat_entry(struct nat_entry *e)

{

kmem_cache_free(nat_entry_slab, e);

}

/* must be locked by nat_tree_lock */

static struct nat_entry *__init_nat_entry(struct f2fs_nm_info *nm_i,

struct nat_entry *ne, struct f2fs_nat_entry *raw_ne, bool no_fail, bool init_dirty)

{

if (no_fail)

f2fs_radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne);

else if (radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne))

return NULL;

if (raw_ne)

node_info_from_raw_nat(&ne->ni, raw_ne);

if (init_dirty) {

INIT_LIST_HEAD(&ne->list);

nm_i->nat_cnt[TOTAL_NAT]++;

return ne;

}

spin_lock(&nm_i->nat_list_lock);

list_add_tail(&ne->list, &nm_i->nat_entries);

spin_unlock(&nm_i->nat_list_lock);

nm_i->nat_cnt[TOTAL_NAT]++;

nm_i->nat_cnt[RECLAIMABLE_NAT]++;

return ne;

}

static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n, bool for_dirty)

{

struct nat_entry *ne;

ne = radix_tree_lookup(&nm_i->nat_root, n);

/*

* for recent accessed nat entry which will not be dirtied soon

* later, move it to tail of lru list.

*/

if (ne && !get_nat_flag(ne, IS_DIRTY) && !for_dirty) {

spin_lock(&nm_i->nat_list_lock);

if (!list_empty(&ne->list))

list_move_tail(&ne->list, &nm_i->nat_entries);

spin_unlock(&nm_i->nat_list_lock);

}

return ne;

}

static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,

nid_t start, unsigned int nr, struct nat_entry **ep)

{

return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);

}

static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)

{

radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));

nm_i->nat_cnt[TOTAL_NAT]--;

nm_i->nat_cnt[RECLAIMABLE_NAT]--;

__free_nat_entry(e);

}

static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i,

struct nat_entry *ne)

{

nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid);

struct nat_entry_set *head;

head = radix_tree_lookup(&nm_i->nat_set_root, set);

if (!head) {

head = f2fs_kmem_cache_alloc(nat_entry_set_slab,

GFP_NOFS, true, NULL);

INIT_LIST_HEAD(&head->entry_list);

INIT_LIST_HEAD(&head->set_list);

head->set = set;

head->entry_cnt = 0;

f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head);

}

return head;

}

static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,

struct nat_entry *ne, bool init_dirty)

{

struct nat_entry_set *head;

bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR;

if (!new_ne)

head = __grab_nat_entry_set(nm_i, ne);

/*

* update entry_cnt in below condition:

* 1. update NEW_ADDR to valid block address;

* 2. update old block address to new one;

*/

if (!new_ne && (get_nat_flag(ne, IS_PREALLOC) ||

!get_nat_flag(ne, IS_DIRTY)))

head->entry_cnt++;

set_nat_flag(ne, IS_PREALLOC, new_ne);

if (get_nat_flag(ne, IS_DIRTY))

goto refresh_list;

nm_i->nat_cnt[DIRTY_NAT]++;

if (!init_dirty)

nm_i->nat_cnt[RECLAIMABLE_NAT]--;

set_nat_flag(ne, IS_DIRTY, true);

refresh_list:

spin_lock(&nm_i->nat_list_lock);

if (new_ne)

list_del_init(&ne->list);

else

list_move_tail(&ne->list, &head->entry_list);

spin_unlock(&nm_i->nat_list_lock);

}

static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i,

struct nat_entry_set *set, struct nat_entry *ne)

{

spin_lock(&nm_i->nat_list_lock);

list_move_tail(&ne->list, &nm_i->nat_entries);

spin_unlock(&nm_i->nat_list_lock);

set_nat_flag(ne, IS_DIRTY, false);

set->entry_cnt--;

nm_i->nat_cnt[DIRTY_NAT]--;

nm_i->nat_cnt[RECLAIMABLE_NAT]++;

}

static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,

nid_t start, unsigned int nr, struct nat_entry_set **ep)

{

return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep,

start, nr);

}

bool f2fs_in_warm_node_list(struct folio *folio)

{

return is_node_folio(folio) && IS_DNODE(folio) && is_cold_node(folio);

}

void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi)

{

spin_lock_init(&sbi->fsync_node_lock);

INIT_LIST_HEAD(&sbi->fsync_node_list);

sbi->fsync_seg_id = 0;

sbi->fsync_node_num = 0;

}

static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi,

struct folio *folio)

{

struct fsync_node_entry *fn;

unsigned long flags;

unsigned int seq_id;

fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab,

GFP_NOFS, true, NULL);

folio_get(folio);

fn->folio = folio;

INIT_LIST_HEAD(&fn->list);

spin_lock_irqsave(&sbi->fsync_node_lock, flags);

list_add_tail(&fn->list, &sbi->fsync_node_list);

fn->seq_id = sbi->fsync_seg_id++;

seq_id = fn->seq_id;

sbi->fsync_node_num++;

spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);

return seq_id;

}

void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio)

{

struct fsync_node_entry *fn;

unsigned long flags;

spin_lock_irqsave(&sbi->fsync_node_lock, flags);

list_for_each_entry(fn, &sbi->fsync_node_list, list) {

if (fn->folio == folio) {

list_del(&fn->list);

sbi->fsync_node_num--;

spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);

kmem_cache_free(fsync_node_entry_slab, fn);

folio_put(folio);

return;

}

}

spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);

f2fs_bug_on(sbi, 1);

}

void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi)

{

unsigned long flags;

spin_lock_irqsave(&sbi->fsync_node_lock, flags);

sbi->fsync_seg_id = 0;

spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);

}

bool f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

struct nat_entry *e;

bool need = false;

f2fs_down_read(&nm_i->nat_tree_lock);

e = __lookup_nat_cache(nm_i, nid, false);

if (e) {

if (!get_nat_flag(e, IS_CHECKPOINTED) &&

!get_nat_flag(e, HAS_FSYNCED_INODE))

need = true;

}

f2fs_up_read(&nm_i->nat_tree_lock);

return need;

}

bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

struct nat_entry *e;

bool is_cp = true;

f2fs_down_read(&nm_i->nat_tree_lock);

e = __lookup_nat_cache(nm_i, nid, false);

if (e && !get_nat_flag(e, IS_CHECKPOINTED))

is_cp = false;

f2fs_up_read(&nm_i->nat_tree_lock);

return is_cp;

}

bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

struct nat_entry *e;

bool need_update = true;

struct f2fs_lock_context lc;

f2fs_down_read_trace(&sbi->node_write, &lc);

f2fs_down_read(&nm_i->nat_tree_lock);

e = __lookup_nat_cache(nm_i, ino, false);

if (e && get_nat_flag(e, HAS_LAST_FSYNC) &&

(get_nat_flag(e, IS_CHECKPOINTED) ||

get_nat_flag(e, HAS_FSYNCED_INODE)))

need_update = false;

f2fs_up_read(&nm_i->nat_tree_lock);

f2fs_up_read_trace(&sbi->node_write, &lc);

return need_update;

}

/* must be locked by nat_tree_lock */

static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid,

struct f2fs_nat_entry *ne)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

struct nat_entry *new, *e;

/* Let's mitigate lock contention of nat_tree_lock during checkpoint */

if (f2fs_rwsem_is_locked(&sbi->cp_global_sem))

return;

new = __alloc_nat_entry(sbi, nid, false);

if (!new)

return;

f2fs_down_write(&nm_i->nat_tree_lock);

e = __lookup_nat_cache(nm_i, nid, false);

if (!e)

e = __init_nat_entry(nm_i, new, ne, false, false);

else

f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) ||

nat_get_blkaddr(e) !=

le32_to_cpu(ne->block_addr) ||

nat_get_version(e) != ne->version);

f2fs_up_write(&nm_i->nat_tree_lock);

if (e != new)

__free_nat_entry(new);

}

static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,

block_t new_blkaddr, bool fsync_done)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

struct nat_entry *e;

struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true);

bool init_dirty = false;

f2fs_down_write(&nm_i->nat_tree_lock);

e = __lookup_nat_cache(nm_i, ni->nid, true);

if (!e) {

init_dirty = true;

e = __init_nat_entry(nm_i, new, NULL, true, true);

copy_node_info(&e->ni, ni);

f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR);

} else if (new_blkaddr == NEW_ADDR) {

/*

* when nid is reallocated,

* previous nat entry can be remained in nat cache.

* So, reinitialize it with new information.

*/

copy_node_info(&e->ni, ni);

f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR);

}

/* let's free early to reduce memory consumption */

if (e != new)

__free_nat_entry(new);

/* sanity check */

f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr);

f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR &&

new_blkaddr == NULL_ADDR);

f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR &&

new_blkaddr == NEW_ADDR);

f2fs_bug_on(sbi, __is_valid_data_blkaddr(nat_get_blkaddr(e)) &&

new_blkaddr == NEW_ADDR);

/* increment version no as node is removed */

if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) {

unsigned char version = nat_get_version(e);

nat_set_version(e, inc_node_version(version));

}

/* change address */

nat_set_blkaddr(e, new_blkaddr);

if (!__is_valid_data_blkaddr(new_blkaddr))

set_nat_flag(e, IS_CHECKPOINTED, false);

__set_nat_cache_dirty(nm_i, e, init_dirty);

/* update fsync_mark if its inode nat entry is still alive */

if (ni->nid != ni->ino)

e = __lookup_nat_cache(nm_i, ni->ino, false);

if (e) {

if (fsync_done && ni->nid == ni->ino)

set_nat_flag(e, HAS_FSYNCED_INODE, true);

set_nat_flag(e, HAS_LAST_FSYNC, fsync_done);

}

f2fs_up_write(&nm_i->nat_tree_lock);

}

int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

int nr = nr_shrink;

if (!f2fs_down_write_trylock(&nm_i->nat_tree_lock))

return 0;

spin_lock(&nm_i->nat_list_lock);

while (nr_shrink) {

struct nat_entry *ne;

if (list_empty(&nm_i->nat_entries))

break;

ne = list_first_entry(&nm_i->nat_entries,

struct nat_entry, list);

list_del(&ne->list);

spin_unlock(&nm_i->nat_list_lock);

__del_from_nat_cache(nm_i, ne);

nr_shrink--;

spin_lock(&nm_i->nat_list_lock);

}

spin_unlock(&nm_i->nat_list_lock);

f2fs_up_write(&nm_i->nat_tree_lock);

return nr - nr_shrink;

}

int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,

struct node_info *ni, bool checkpoint_context)

{

struct f2fs_nm_info *nm_i = NM_I(sbi);

struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);

struct f2fs_journal *journal = curseg->journal;

nid_t start_nid = START_NID(nid);

struct f2fs_nat_block *nat_blk;

struct folio *folio = NULL;

struct f2fs_nat_entry ne;

struct nat_entry *e;

pgoff_t index;

int i;

bool need_cache = true;

ni->flag = 0;

ni->nid = nid;

retry:

/* Check nat cache */

f2fs_down_read(&nm_i->nat_tree_lock);

e = __lookup_nat_cache(nm_i, nid, false);

if (e) {

ni->ino = nat_get_ino(e);

ni->blk_addr = nat_get_blkaddr(e);

ni->version = nat_get_version(e);

f2fs_up_read(&nm_i->nat_tree_lock);

if (IS_ENABLED(CONFIG_F2FS_CHECK_FS)) {

need_cache = false;

goto sanity_check;

}

return 0;

}

/*

* Check current segment summary by trying to grab journal_rwsem first.

* This sem is on the critical path on the checkpoint requiring the above

* nat_tree_lock. Therefore, we should retry, if we failed to grab here

* while not bothering checkpoint.

*/

if (!f2fs_rwsem_is_locked(&sbi->cp_global_sem) || checkpoint_context) {

down_read(&curseg->journal_rwsem);

} else if (f2fs_rwsem_is_contended(&nm_i->nat_tree_lock) ||

!down_read_trylock(&curseg->journal_rwsem)) {

f2fs_up_read(&nm_i->nat_tree_lock);

goto retry;

}

i = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 0);

if (i >= 0) {

ne = nat_in_journal(journal, i);

node_info_from_raw_nat(ni, &ne);

}

up_read(&curseg->journal_rwsem);

if (i >= 0) {

f2fs_up_read(&nm_i->nat_tree_lock);

goto sanity_check;

}

/* Fill node_info from nat page */

index = current_nat_addr(sbi, nid);

f2fs_up_read(&nm_i->nat_tree_lock);

folio = f2fs_get_meta_folio(sbi, index);

if (IS_ERR(folio))

return PTR_ERR(folio);

nat_blk = folio_address(folio);

ne = nat_blk->entries[nid - start_nid];

node_info_from_raw_nat(ni, &ne);

f2fs_folio_put(folio, true);

sanity_check:

if (__is_valid_data_blkaddr(ni->blk_addr) &&

!f2fs_is_valid_blkaddr(sbi, ni->blk_addr,

DATA_GENERIC_ENHANCE)) {

set_sbi_flag(sbi, SBI_NEED_FSCK);

f2fs_err_ratelimited(sbi,

"f2fs_get_node_info of %pS: inconsistent nat entry, "

"ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u",

__builtin_return_address(0),

ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag);

f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT);

return -EFSCORRUPTED;

}

if (unlikely(f2fs_quota_file(sbi, ni->nid) &&

!__is_valid_data_blkaddr(ni->blk_addr))) {

set_sbi_flag(sbi, SBI_NEED_FSCK);

f2fs_err_ratelimited(sbi,

"f2fs_get_node_info of %pS: inconsistent nat entry from qf_ino, "

"ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u",

__builtin_return_address(0),

ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag);

f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT);

}

/* cache nat entry */

if (need_cache)

cache_nat_entry(sbi, nid, &ne);

return 0;

}

/*

* readahead MAX_RA_NODE number of node pages.

*/

static void f2fs_ra_node_pages(struct folio *parent, int start, int n)

{

struct f2fs_sb_info *sbi = F2FS_F_SB(parent);

struct blk_plug plug;

int i, end;

nid_t nid;

blk_start_plug(&plug);

/* Then, try readahead for siblings of the desired node */

end = start + n;

end = min(end, (int)NIDS_PER_BLOCK);

for (i = start; i < end; i++) {

nid = get_nid(parent, i, false);

f2fs_ra_node_page(sbi, nid);

}

blk_finish_plug(&plug);

}

pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs)

{

const long direct_index = ADDRS_PER_INODE(dn->inode);

const long direct_blks = ADDRS_PER_BLOCK(dn->inode);

const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) * NIDS_PER_BLOCK;

unsigned int skipped_unit = ADDRS_PER_BLOCK(dn->inode);

int cur_level = dn->cur_level;

int max_level = dn->max_level;

pgoff_t base = 0;

if (!dn->max_level)

return pgofs + 1;

while (max_level-- > cur_level)

skipped_unit *= NIDS_PER_BLOCK;

switch (dn->max_level) {

case 3:

base += 2 * indirect_blks;

fallthrough;

case 2:

base += 2 * direct_blks;

fallthrough;

case 1:

base += direct_index;

break;

default:

f2fs_bug_on(F2FS_I_SB(dn->inode), 1);

}

return ((pgofs - base) / skipped_unit + 1) * skipped_unit + base;

}

/*

* The maximum depth is four.

* Offset[0] will have raw inode offset.

*/

static int get_node_path(struct inode *inode, long block,

int offset[4], unsigned int noffset[4])

{

const long direct_index = ADDRS_PER_INODE(inode);

const long direct_blks = ADDRS_PER_BLOCK(inode);

const long dptrs_per_blk = NIDS_PER_BLOCK;

const long indirect_blks = ADDRS_PER_BLOCK(inode) * NIDS_PER_BLOCK;

const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;

int n = 0;

int level = 0;

noffset[0] = 0;

if (block < direct_index) {

offset[n] = block;

goto got;

}

block -= direct_index;

if (block < direct_blks) {

offset[n++] = NODE_DIR1_BLOCK;

noffset[n] = 1;

offset[n] = block;

level = 1;

goto got;

}

block -= direct_blks;

if (block < direct_blks) {

offset[n++] = NODE_DIR2_BLOCK;

noffset[n] = 2;

offset[n] = block;

level = 1;

goto got;

}

block -= direct_blks;

if (block < indirect_blks) {

offset[n++] = NODE_IND1_BLOCK;

noffset[n] = 3;

offset[n++] = block / direct_blks;

noffset[n] = 4 + offset[n - 1];

offset[n] = block % direct_blks;

level = 2;

goto got;

}

block -= indirect_blks;

if (block < indirect_blks) {

offset[n++] = NODE_IND2_BLOCK;

noffset[n] = 4 + dptrs_per_blk;

offset[n++] = block / direct_blks;

noffset[n] = 5 + dptrs_per_blk + offset[n - 1];

offset[n] = block % direct_blks;

level = 2;

goto got;

}

block -= indirect_blks;

if (block < dindirect_blks) {

offset[n++] = NODE_DIND_BLOCK;

noffset[n] = 5 + (dptrs_per_blk * 2);

offset[n++] = block / indirect_blks;

noffset[n] = 6 + (dptrs_per_blk * 2) +

offset[n - 1] * (dptrs_per_blk + 1);

offset[n++] = (block / direct_blks) % dptrs_per_blk;

noffset[n] = 7 + (dptrs_per_blk * 2) +

offset[n - 2] * (dptrs_per_blk + 1) +

offset[n - 1];

offset[n] = block % direct_blks;

level = 3;

goto got;

} else {

return -E2BIG;

}

got:

return level;

}

static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start);

/*

* Caller should call f2fs_put_dnode(dn).

* Also, it should grab and release a rwsem by calling f2fs_lock_op() and

* f2fs_unlock_op() only if mode is set with ALLOC_NODE.

*/

int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)

{

struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);

struct folio *nfolio[4];

struct folio *parent = NULL;

int offset[4];

unsigned int noffset[4];

nid_t nids[4];

int level, i = 0;

int err = 0;

level = get_node_path(dn->inode, index, offset, noffset);

if (level < 0)

return level;

nids[0] = dn->inode->i_ino;

if (!dn->inode_folio) {

nfolio[0] = f2fs_get_inode_folio(sbi, nids[0]);

if (IS_ERR(nfolio[0]))

return PTR_ERR(nfolio[0]);

} else {

nfolio[0] = dn->inode_folio;

}

/* if inline_data is set, should not report any block indices */

if (f2fs_has_inline_data(dn->inode) && index) {

err = -ENOENT;

f2fs_folio_put(nfolio[0], true);

goto release_out;

}

parent = nfolio[0];

if (level != 0)

nids[1] = get_nid(parent, offset[0], true);

dn->inode_folio = nfolio[0];

dn->inode_folio_locked = true;

/* get indirect or direct nodes */

for (i = 1; i <= level; i++) {

bool done = false;

if (nids[i] && nids[i] == dn->inode->i_ino) {

err = -EFSCORRUPTED;

f2fs_err_ratelimited(sbi,

"inode mapping table is corrupted, run fsck to fix it, "

"ino:%llu, nid:%u, level:%d, offset:%d",

dn->inode->i_ino, nids[i], level, offset[level]);

set_sbi_flag(sbi, SBI_NEED_FSCK);

goto release_pages;

}

if (!nids[i] && mode == ALLOC_NODE) {

/* alloc new node */

if (!f2fs_alloc_nid(sbi, &(nids[i]))) {

err = -ENOSPC;

goto release_pages;

}

dn->nid = nids[i];

nfolio[i] = f2fs_new_node_folio(dn, noffset[i]);

if (IS_ERR(nfolio[i])) {

f2fs_alloc_nid_failed(sbi, nids[i]);

err = PTR_ERR(nfolio[i]);

goto release_pages;

}

set_nid(parent, offset[i - 1], nids[i], i == 1);

f2fs_alloc_nid_done(sbi, nids[i]);

done = true;

} else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {

nfolio[i] = f2fs_get_node_folio_ra(parent, offset[i - 1]);

if (IS_ERR(nfolio[i])) {

err = PTR_ERR(nfolio[i]);

goto release_pages;

}

done = true;

}

if (i == 1) {

dn->inode_folio_locked = false;

folio_unlock(parent);

} else {

f2fs_folio_put(parent, true);

}

if (!done) {

nfolio[i] = f2fs_get_node_folio(sbi, nids[i],

NODE_TYPE_NON_INODE);

if (IS_ERR(nfolio[i])) {

err = PTR_ERR(nfolio[i]);

f2fs_folio_put(nfolio[0], false);

goto release_out;

}

}

if (i < level) {

parent = nfolio[i];

nids[i + 1] = get_nid(parent, offset[i], false);

}

}

dn->nid = nids[level];

dn->ofs_in_node = offset[level];

dn->node_folio = nfolio[level];

dn->data_blkaddr = f2fs_data_blkaddr(dn);

if (is_inode_flag_set(dn->inode, FI_COMPRESSED_FILE) &&

f2fs_sb_has_readonly(sbi)) {

unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size;

unsigned int ofs_in_node = dn->ofs_in_node;

pgoff_t fofs = index;

unsigned int c_len;

block_t blkaddr;

/* should align fofs and ofs_in_node to cluster_size */

if (fofs % cluster_size) {

fofs = round_down(fofs, cluster_size);

ofs_in_node = round_down(ofs_in_node, cluster_size);

}

c_len = f2fs_cluster_blocks_are_contiguous(dn, ofs_in_node);

if (!c_len)

goto out;

blkaddr = data_blkaddr(dn->inode, dn->node_folio, ofs_in_node);

if (blkaddr == COMPRESS_ADDR)

blkaddr = data_blkaddr(dn->inode, dn->node_folio,

ofs_in_node + 1);

f2fs_update_read_extent_tree_range_compressed(dn->inode,

fofs, blkaddr, cluster_size, c_len);

}

out:

return 0;

release_pages:

f2fs_folio_put(parent, true);

if (i > 1)

f2fs_folio_put(nfolio[0], false);

release_out:

dn->inode_folio = NULL;

dn->node_folio = NULL;

if (err == -ENOENT) {

dn->cur_level = i;

dn->max_level = level;

dn->ofs_in_node = offset[level];

}

return err;

}

static int truncate_node(struct dnode_of_data *dn)

{

struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);

struct node_info ni;

int err;

pgoff_t index;

err = f2fs_get_node_info(sbi, dn->nid, &ni, false);

if (err)

return err;

if (ni.blk_addr != NEW_ADDR &&

!f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) {

f2fs_err_ratelimited(sbi,

"nat entry is corrupted, run fsck to fix it, ino:%u, "

"nid:%u, blkaddr:%u", ni.ino, ni.nid, ni.blk_addr);

set_sbi_flag(sbi, SBI_NEED_FSCK);

f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT);

return -EFSCORRUPTED;

}

/* Deallocate node address */

f2fs_invalidate_blocks(sbi, ni.blk_addr, 1);

dec_valid_node_count(sbi, dn->inode, dn->nid == dn->inode->i_ino);

set_node_addr(sbi, &ni, NULL_ADDR, false);

if (dn->nid == dn->inode->i_ino) {

f2fs_remove_orphan_inode(sbi, dn->nid);

dec_valid_inode_count(sbi);

f2fs_inode_synced(dn->inode);

}

clear_node_folio_dirty(dn->node_folio);

set_sbi_flag(sbi, SBI_IS_DIRTY);

index = dn->node_folio->index;

f2fs_folio_put(dn->node_folio, true);

invalidate_mapping_pages(NODE_MAPPING(sbi),

index, index);

dn->node_folio = NULL;

trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);

return 0;

}